43 add robots and sitemap

README.md

@@ -67,7 +67,16 @@ ruby generate_browse_pages.rb --books ./data/books.json --output-dir ./output
 ruby generate_static_pages.rb --output-dir ./output
 ```
 
-**5. Deploy to S3**
+**5. Generate sitemap.xml and robots.txt**
+
+```bash
+ruby generate_sitemap.rb --books ./data/books.json --output-dir ./output
+```
+
+The sitemap lists the static/browse pages and the public book pages as absolute
+URLs rooted at `https://publishing.cdlib.org`.
+
+**6. Deploy to S3**
 
 ```bash
 # Dry run first to review changes

deploy.sh

@@ -9,7 +9,8 @@
 #   ./deploy.sh --profile <aws-profile> --env stg --execute
 
 # Syncs:
-#   output/  -> s3://ucpec/<env>/ucpressebooks/
+#   output/ -> s3://ucpec/<env>/ucpressebooks/
+#   output/robots.txt -> s3://ucpec/<env>/robots.txt (domain root)
 
 set -euo pipefail
 
@@ -47,6 +48,7 @@ SYNC_ARGS=(
   --profile "$PROFILE"
   --delete
   --exclude ".DS_Store"
+  --exclude "robots.txt"
 )
 
 if $DRY_RUN; then
@@ -62,6 +64,18 @@ echo ""
 echo "Syncing output/ ..."
 aws s3 sync "${OUTPUT_DIR}/" "${S3_BASE}/" "${SYNC_ARGS[@]}"
 
+# robots.txt is only honored by crawlers at the domain root
+ROBOTS_SRC="${OUTPUT_DIR}/robots.txt"
+if [[ -f "$ROBOTS_SRC" && "$ENV" == "prd" ]]; then
+  echo ""
+  echo "Placing robots.txt at site root ..."
+  CP_ARGS=(--profile "$PROFILE")
+  if $DRY_RUN; then
+    CP_ARGS+=(--dryrun)
+  fi
+  aws s3 cp "$ROBOTS_SRC" "s3://${BUCKET}/${ENV}/robots.txt" "${CP_ARGS[@]}"
+fi
+
 echo ""
 if $DRY_RUN; then
   echo "Dry run complete. Run with --execute to deploy."

generate_browse_pages.rb

@@ -21,12 +21,6 @@ def pub_info(book)
   [book["publisher"], book["year"]].compact.reject(&:empty?).join(", ")
 end
 
-# Converts a string to a URL/HTML-ID-safe slug
-# (e.g. "Cinema and Performance Arts" -> "cinema-and-performance-arts")
-def slugify(str)
-  str.downcase.gsub(/[^a-z0-9]+/, "-").delete_prefix("-").delete_suffix("-")
-end
-
 options = { books: "./data/books.json", output_dir: "./output" }
 
 OptionParser.new do |opts|

generate_sitemap.rb

@@ -0,0 +1,78 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Generates sitemap.xml and robots.txt into the output directory
+# The sitemap lists only publicly reachable URLs (access-restricted book and error pages are excluded)
+
+# Usage:
+# ruby generate_sitemap.rb --books ./data/books.json --output-dir ./output
+
+require "json"
+require "optparse"
+require "pathname"
+require_relative "shared_page_helpers"
+
+# Static pages that should be discoverable by crawlers
+STATIC_PATHS = %w[
+  index.html
+  about.html
+  help.html
+  browse_subject.html
+  browse_title.html
+  browse_author.html
+].freeze
+
+def public_book_path(book)
+  "#{BASE_PATH}public/book/#{slugify(book["title"])}.html"
+end
+
+# Minimal XML escaping for the <loc> element
+def xml_escape(str)
+  str.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;")
+end
+
+options = { books: "./data/books.json", output_dir: "./output" }
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: ruby generate_sitemap.rb [options]"
+  opts.on("--books FILE",     "Path to books.json cache") { |v| options[:books] = v }
+  opts.on("--output-dir DIR", "Directory to write files into") { |v| options[:output_dir] = v }
+end.parse!
+
+unless File.exist?(options[:books])
+  warn "books.json not found at #{options[:books]}"
+  exit 1
+end
+
+all_books = JSON.parse(File.read(options[:books], encoding: "UTF-8"))
+public_books = all_books.select { |b| b["public"] }
+
+output_dir = Pathname.new(options[:output_dir])
+output_dir.mkpath
+
+# Build the ordered list of absolute URLs
+urls = STATIC_PATHS.map { |p| absolute_url("#{BASE_PATH}#{p}") }
+urls += public_books.map { |b| absolute_url(public_book_path(b)) }
+
+# sitemap.xml
+sitemap = +%(<?xml version="1.0" encoding="UTF-8"?>\n)
+sitemap << %(<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n)
+urls.each { |url| sitemap << "  <url>\n    <loc>#{xml_escape(url)}</loc>\n  </url>\n" }
+sitemap << "</urlset>\n"
+
+output_dir.join("sitemap.xml").write(sitemap)
+warn "Wrote sitemap.xml (#{urls.size} URLs: #{STATIC_PATHS.size} static + #{public_books.size} public books)"
+
+# robots.txt
+robots = <<~ROBOTS
+  User-agent: *
+  Allow: #{BASE_PATH}
+  Disallow: #{BASE_PATH}uc/
+  Crawl-delay: 5
+  Sitemap: #{absolute_url("#{BASE_PATH}sitemap.xml")}
+ROBOTS
+
+output_dir.join("robots.txt").write(robots)
+warn "Wrote robots.txt"
+
+warn "\nDone. 2 files written to #{output_dir}/"

shared_page_helpers.rb

@@ -12,6 +12,18 @@
 BRAND_NAME = "UC Press E-Books Collection, 1982-2004"
 TEMPLATES  = Pathname.new(__dir__).join("templates")
 BASE_PATH  = "/ucpressebooks/"
+SITE_URL   = "https://publishing.cdlib.org"
+
+# Joins an absolute site path (e.g. "/ucpressebooks/index.html") onto SITE_URL
+def absolute_url(path)
+  "#{SITE_URL}#{path.start_with?('/') ? path : "/#{path}"}"
+end
+
+# Converts a string to a URL/HTML-ID-safe slug
+# (e.g. "Cinema and Performance Arts" -> "cinema-and-performance-arts")
+def slugify(str)
+  str.downcase.gsub(/[^a-z0-9]+/, "-").delete_prefix("-").delete_suffix("-")
+end
 
 def render(template_path, b)
   ERB.new(File.read(template_path), trim_mode: "-").result(b)