From b363503c55da46a9acf7385fad12357d555d4b39 Mon Sep 17 00:00:00 2001 From: nicktahani Date: Tue, 11 Aug 2026 15:24:37 -0700 Subject: [PATCH 1/6] add sitemap and robots.txt generation script --- generate_sitemap.rb | 77 ++++++++++++++++++++++++++++++++++++++++++ shared_page_helpers.rb | 12 +++++++ 2 files changed, 89 insertions(+) create mode 100644 generate_sitemap.rb diff --git a/generate_sitemap.rb b/generate_sitemap.rb new file mode 100644 index 0000000..e0ed027 --- /dev/null +++ b/generate_sitemap.rb @@ -0,0 +1,77 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Generates sitemap.xml and robots.txt into the output directory +# The sitemap lists only publicly reachable URLs (access-restricted book and error pages are excluded) + +# Usage: +# ruby generate_sitemap.rb --books ./data/books.json --output-dir ./output + +require "json" +require "optparse" +require "pathname" +require_relative "shared_page_helpers" + +# Static pages that should be discoverable by crawlers +STATIC_PATHS = %w[ + index.html + about.html + help.html + browse_subject.html + browse_title.html + browse_author.html +].freeze + +def public_book_path(book) + "#{BASE_PATH}public/book/#{slugify(book["title"])}.html" +end + +# Minimal XML escaping for the element +def xml_escape(str) + str.gsub("&", "&").gsub("<", "<").gsub(">", ">") +end + +options = { books: "./data/books.json", output_dir: "./output" } + +OptionParser.new do |opts| + opts.banner = "Usage: ruby generate_sitemap.rb [options]" + opts.on("--books FILE", "Path to books.json cache") { |v| options[:books] = v } + opts.on("--output-dir DIR", "Directory to write files into") { |v| options[:output_dir] = v } +end.parse! + +unless File.exist?(options[:books]) + warn "books.json not found at #{options[:books]}" + exit 1 +end + +all_books = JSON.parse(File.read(options[:books], encoding: "UTF-8")) +public_books = all_books.select { |b| b["public"] } + +output_dir = Pathname.new(options[:output_dir]) +output_dir.mkpath + +# Build the ordered list of absolute URLs +urls = STATIC_PATHS.map { |p| absolute_url("#{BASE_PATH}#{p}") } +urls += public_books.map { |b| absolute_url(public_book_path(b)) } + +# sitemap.xml +sitemap = +%(\n) +sitemap << %(\n) +urls.each { |url| sitemap << " \n #{xml_escape(url)}\n \n" } +sitemap << "\n" + +output_dir.join("sitemap.xml").write(sitemap) +warn "Wrote sitemap.xml (#{urls.size} URLs: #{STATIC_PATHS.size} static + #{public_books.size} public books)" + +# robots.txt +robots = <<~ROBOTS + User-agent: * + Allow: #{BASE_PATH} + Disallow: #{BASE_PATH}uc/ + Sitemap: #{absolute_url("#{BASE_PATH}sitemap.xml")} +ROBOTS + +output_dir.join("robots.txt").write(robots) +warn "Wrote robots.txt" + +warn "\nDone. 2 files written to #{output_dir}/" diff --git a/shared_page_helpers.rb b/shared_page_helpers.rb index 9e5d6e6..b53a14a 100644 --- a/shared_page_helpers.rb +++ b/shared_page_helpers.rb @@ -12,6 +12,18 @@ BRAND_NAME = "UC Press E-Books Collection, 1982-2004" TEMPLATES = Pathname.new(__dir__).join("templates") BASE_PATH = "/ucpressebooks/" +SITE_URL = "https://publishing.cdlib.org" + +# Joins an absolute site path (e.g. "/ucpressebooks/index.html") onto SITE_URL +def absolute_url(path) + "#{SITE_URL}#{path.start_with?('/') ? path : "/#{path}"}" +end + +# Converts a string to a URL/HTML-ID-safe slug +# (e.g. "Cinema and Performance Arts" -> "cinema-and-performance-arts") +def slugify(str) + str.downcase.gsub(/[^a-z0-9]+/, "-").delete_prefix("-").delete_suffix("-") +end def render(template_path, b) ERB.new(File.read(template_path), trim_mode: "-").result(b) From 7ec1a976b0f8550272298710857fafe672611cbd Mon Sep 17 00:00:00 2001 From: nicktahani Date: Tue, 11 Aug 2026 15:25:28 -0700 Subject: [PATCH 2/6] remove helper method (now in shared helpers file) --- generate_browse_pages.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/generate_browse_pages.rb b/generate_browse_pages.rb index 81d1ab6..b418f94 100755 --- a/generate_browse_pages.rb +++ b/generate_browse_pages.rb @@ -21,12 +21,6 @@ def pub_info(book) [book["publisher"], book["year"]].compact.reject(&:empty?).join(", ") end -# Converts a string to a URL/HTML-ID-safe slug -# (e.g. "Cinema and Performance Arts" -> "cinema-and-performance-arts") -def slugify(str) - str.downcase.gsub(/[^a-z0-9]+/, "-").delete_prefix("-").delete_suffix("-") -end - options = { books: "./data/books.json", output_dir: "./output" } OptionParser.new do |opts| From 72ae164514cf0587f155036c3cbb16f66089257a Mon Sep 17 00:00:00 2001 From: nicktahani Date: Tue, 11 Aug 2026 15:26:10 -0700 Subject: [PATCH 3/6] place robots.txt at site root --- deploy.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/deploy.sh b/deploy.sh index 8108db9..6cc60e1 100755 --- a/deploy.sh +++ b/deploy.sh @@ -9,7 +9,8 @@ # ./deploy.sh --profile --env stg --execute # Syncs: -# output/ -> s3://ucpec//ucpressebooks/ +# output/ -> s3://ucpec//ucpressebooks/ +# output/robots.txt -> s3://ucpec//robots.txt (domain root) set -euo pipefail @@ -47,6 +48,7 @@ SYNC_ARGS=( --profile "$PROFILE" --delete --exclude ".DS_Store" + --exclude "robots.txt" ) if $DRY_RUN; then @@ -62,6 +64,18 @@ echo "" echo "Syncing output/ ..." aws s3 sync "${OUTPUT_DIR}/" "${S3_BASE}/" "${SYNC_ARGS[@]}" +# robots.txt is only honored by crawlers at the domain root +ROBOTS_SRC="${OUTPUT_DIR}/robots.txt" +if [[ -f "$ROBOTS_SRC" && "$ENV" == "prd" ]]; then + echo "" + echo "Placing robots.txt at site root ..." + CP_ARGS=(--profile "$PROFILE") + if $DRY_RUN; then + CP_ARGS+=(--dryrun) + fi + aws s3 cp "$ROBOTS_SRC" "s3://${BUCKET}/${ENV}/robots.txt" "${CP_ARGS[@]}" +fi + echo "" if $DRY_RUN; then echo "Dry run complete. Run with --execute to deploy." From f3aaea5c16f3f3313fa46bf2a7610b850ae10a5d Mon Sep 17 00:00:00 2001 From: nicktahani Date: Tue, 11 Aug 2026 15:26:58 -0700 Subject: [PATCH 4/6] update readme to include instructions for generating sitemap and robots.txt --- README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5bc6f7e..409eefe 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,16 @@ ruby generate_browse_pages.rb --books ./data/books.json --output-dir ./output ruby generate_static_pages.rb --output-dir ./output ``` -**5. Deploy to S3** +**5. Generate sitemap.xml and robots.txt** + +```bash +ruby generate_sitemap.rb --books ./data/books.json --output-dir ./output +``` + +The sitemap lists the static/browse pages and the public book pages as absolute +URLs rooted at `https://publishing.cdlib.org`. + +**6. Deploy to S3** ```bash # Dry run first to review changes From ec8b4df302d49fc5c72ef471411123e2c3164524 Mon Sep 17 00:00:00 2001 From: nicktahani Date: Thu, 20 Aug 2026 13:25:12 -0700 Subject: [PATCH 5/6] add crawl-delay to robots --- generate_sitemap.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/generate_sitemap.rb b/generate_sitemap.rb index e0ed027..9993eea 100644 --- a/generate_sitemap.rb +++ b/generate_sitemap.rb @@ -68,6 +68,7 @@ def xml_escape(str) User-agent: * Allow: #{BASE_PATH} Disallow: #{BASE_PATH}uc/ + Crawl-delay: 5 Sitemap: #{absolute_url("#{BASE_PATH}sitemap.xml")} ROBOTS From eb35610b992f8a007f36d57a0454514b21024843 Mon Sep 17 00:00:00 2001 From: nicktahani Date: Thu, 20 Aug 2026 13:31:38 -0700 Subject: [PATCH 6/6] rubocop --- generate_sitemap.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 generate_sitemap.rb diff --git a/generate_sitemap.rb b/generate_sitemap.rb old mode 100644 new mode 100755 index 9993eea..c02eb71 --- a/generate_sitemap.rb +++ b/generate_sitemap.rb @@ -44,7 +44,7 @@ def xml_escape(str) exit 1 end -all_books = JSON.parse(File.read(options[:books], encoding: "UTF-8")) +all_books = JSON.parse(File.read(options[:books], encoding: "UTF-8")) public_books = all_books.select { |b| b["public"] } output_dir = Pathname.new(options[:output_dir])