This domain is for use in illustrative examples in documents. You may use this\n domain in literature without prior coordination or asking for permission.
", + "content": "normal" + }, + "type": "text" + } + ], + "_LargestContentfulPaintType": "text", + "_LargestContentfulPaintNodeType": "P", + "_chromeUserTiming": [ + { + "name": "navigationStart", + "time": 13 + }, + { + "name": "fetchStart", + "time": 22 + }, + { + "name": "navigationStart", + "time": 57 + }, + { + "name": "domLoading", + "time": 57 + }, + { + "name": "responseEnd", + "time": 58 + }, + { + "name": "domInteractive", + "time": 58 + }, + { + "name": "domContentLoadedEventStart", + "time": 58 + }, + { + "name": "domContentLoadedEventEnd", + "time": 58 + }, + { + "name": "domComplete", + "time": 58 + }, + { + "name": "responseEnd", + "time": 195 + }, + { + "name": "unloadEventStart", + "time": 204 + }, + { + "name": "unloadEventEnd", + "time": 204 + }, + { + "name": "markAsMainFrame", + "time": 204 + }, + { + "name": "commitNavigationEnd", + "time": 205 + }, + { + "name": "domLoading", + "time": 205 + }, + { + "name": "domInteractive", + "time": 217 + }, + { + "name": "domContentLoadedEventStart", + "time": 217 + }, + { + "name": "domContentLoadedEventEnd", + "time": 217 + }, + { + "name": "domComplete", + "time": 218 + }, + { + "name": "loadEventStart", + "time": 218 + }, + { + "name": "loadEventEnd", + "time": 218 + }, + { + "name": "firstPaint", + "time": 300 + }, + { + "name": "firstContentfulPaint", + "time": 300 + }, + { + "name": "firstMeaningfulPaintCandidate", + "time": 300 + }, + { + "name": "firstMeaningfulPaint", + "time": 300 + }, + { + "name": "firstMeaningfulPaint", + "time": 300 + }, + { + "name": "LargestTextPaint", + "time": 300 + }, + { + "name": "LargestContentfulPaint", + "time": 300 + }, + { + "name": "TotalLayoutShift", + "value": 0 + }, + { + "name": "CumulativeLayoutShift", + "value": 0 + } + ], + "_chromeUserTiming.navigationStart": 57, + "_chromeUserTiming.fetchStart": 22, + "_chromeUserTiming.domLoading": 205, + "_chromeUserTiming.responseEnd": 195, + "_chromeUserTiming.domInteractive": 217, + "_chromeUserTiming.domContentLoadedEventStart": 217, + "_chromeUserTiming.domContentLoadedEventEnd": 217, + "_chromeUserTiming.domComplete": 218, + "_chromeUserTiming.unloadEventStart": 204, + "_chromeUserTiming.unloadEventEnd": 204, + "_chromeUserTiming.markAsMainFrame": 204, + "_chromeUserTiming.commitNavigationEnd": 205, + "_chromeUserTiming.loadEventStart": 218, + "_chromeUserTiming.loadEventEnd": 218, + "_chromeUserTiming.firstPaint": 300, + "_chromeUserTiming.firstContentfulPaint": 300, + "_chromeUserTiming.firstMeaningfulPaintCandidate": 300, + "_chromeUserTiming.firstMeaningfulPaint": 300, + "_chromeUserTiming.LargestTextPaint": 300, + "_chromeUserTiming.LargestContentfulPaint": 300, + "_chromeUserTiming.TotalLayoutShift": 0, + "_chromeUserTiming.CumulativeLayoutShift": 0, + "_blinkFeatureFirstUsed": { + "Features": { + "973": { + "name": "CSSAtRuleMedia", + "firstUsed": 215.369 + }, + "1075": { + "name": "V8SloppyMode", + "firstUsed": 212.591 + }, + "1441": { + "name": "SecureContextCheckPassed", + "firstUsed": 207.019 + }, + "1910": { + "name": "CrossOriginMainFrameNulledNameAccessed", + "firstUsed": 207.032 + }, + "2188": { + "name": "PerformanceObserverForWindow", + "firstUsed": 213.196 + }, + "2203": { + "name": "HasIDClassTagAttribute", + "firstUsed": 215.178 + }, + "2219": { + "name": "V8Element_GetBoundingClientRect_Method", + "firstUsed": 314.825 + }, + "2627": { + "name": "PageLifeCycleFreeze", + "firstUsed": 209.275 + }, + "2782": { + "name": "LayoutJankExplicitlyRequested", + "firstUsed": 213.276 + }, + "2927": { + "name": "LargestContentfulPaintExplicitlyRequested", + "firstUsed": 213.251 + }, + "3004": { + "name": "PerformanceObserverBufferedFlag", + "firstUsed": 213.257 + } + }, + "CSSFeatures": { + "2": { + "name": "CSSPropertyColor", + "firstUsed": 215.353 + }, + "6": { + "name": "CSSPropertyFontFamily", + "firstUsed": 215.295 + }, + "24": { + "name": "CSSPropertyBackgroundColor", + "firstUsed": 215.198 + }, + "53": { + "name": "CSSPropertyBorderRadius", + "firstUsed": 215.319 + }, + "68": { + "name": "CSSPropertyBoxShadow", + "firstUsed": 215.33 + }, + "89": { + "name": "CSSPropertyMargin", + "firstUsed": 215.26 + }, + "109": { + "name": "CSSPropertyPadding", + "firstUsed": 215.267 + }, + "129": { + "name": "CSSPropertyTextDecoration", + "firstUsed": 215.363 + }, + "161": { + "name": "CSSPropertyWidth", + "firstUsed": 215.307 + } + }, + "AnimatedCSSFeatures": {} + }, + "_TTIMeasurementEnd": 3330, + "_LastInteractive": 300, + "_testID": "230509_Dx20W_FMHK5", + "_run": 1, + "_step": 1, + "_effectiveBps": 13500, + "_domTime": 0, + "_aft": 0, + "_titleTime": 62, + "_domLoading": 0, + "_server_rtt": 0, + "_CrUX": { + "key": { + "formFactor": "DESKTOP", + "url": "https://www.example.com/" + }, + "metrics": { + "interaction_to_next_paint": { + "histogram": [ + { + "start": 0, + "end": 200, + "density": 0.9835904971834437 + }, + { + "start": 200, + "end": 500, + "density": 0.00979671809943669 + }, + { + "start": 500, + "density": 0.006612784717119725 + } + ], + "percentiles": { + "p75": 15 + } + }, + "largest_contentful_paint": { + "histogram": [ + { + "start": 0, + "end": 2500, + "density": 0.9567035670356704 + }, + { + "start": 2500, + "end": 4000, + "density": 0.028044280442804437 + }, + { + "start": 4000, + "density": 0.01525215252152517 + } + ], + "percentiles": { + "p75": 1088 + } + }, + "cumulative_layout_shift": { + "histogram": [ + { + "start": "0.00", + "end": "0.10", + "density": 0.9982784062961141 + }, + { + "start": "0.10", + "end": "0.25", + "density": 0.0007378258730939499 + }, + { + "start": "0.25", + "density": 0.0009837678307919328 + } + ], + "percentiles": { + "p75": "0.00" + } + }, + "experimental_time_to_first_byte": { + "histogram": [ + { + "start": 0, + "end": 800, + "density": 0.7510068704098558 + }, + { + "start": 800, + "end": 1800, + "density": 0.2020848140251126 + }, + { + "start": 1800, + "density": 0.04690831556503168 + } + ], + "percentiles": { + "p75": 797 + } + }, + "first_contentful_paint": { + "histogram": [ + { + "start": 0, + "end": 1800, + "density": 0.9130541871921183 + }, + { + "start": 1800, + "end": 3000, + "density": 0.05714285714285715 + }, + { + "start": 3000, + "density": 0.02980295566502464 + } + ], + "percentiles": { + "p75": 1089 + } + }, + "first_input_delay": { + "histogram": [ + { + "start": 0, + "end": 100, + "density": 0.9848693259972491 + }, + { + "start": 100, + "end": 300, + "density": 0.007794589637780836 + }, + { + "start": 300, + "density": 0.007336084364970164 + } + ], + "percentiles": { + "p75": 2 + } + } + }, + "collectionPeriod": { + "firstDate": { + "year": 2023, + "month": 4, + "day": 13 + }, + "lastDate": { + "year": 2023, + "month": 5, + "day": 10 + } + } + }, + "_lighthouse.Performance": 0.99, + "_lighthouse.Performance.first-contentful-paint": 780.895, + "_lighthouse.Performance.largest-contentful-paint": 780.895, + "_lighthouse.Performance.total-blocking-time": 0, + "_lighthouse.Performance.cumulative-layout-shift": 0, + "_lighthouse.Performance.speed-index": 773, + "_lighthouse.Accessibility": 0.91, + "_lighthouse.BestPractices": 0.92, + "_lighthouse.SEO": 0.89, + "_lighthouse.PWA": 0.25, + "_edge-processed": true, + "_consoleLog": [ + { + "source": "network", + "level": "error", + "text": "Failed to load resource: the server responded with a status of 404 ()", + "timestamp": 1683928804036.864, + "url": "https://www.example.com/favicon.ico", + "networkRequestId": "129591.2" + } + ] +} diff --git a/static/well-known/atproto-did b/public/static/well-known/atproto-did similarity index 100% rename from static/well-known/atproto-did rename to public/static/well-known/atproto-did diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 6268824a..00000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -Flask==3.1.3 -flask-talisman==1.1.0 -google-cloud-storage==3.12.0 -gunicorn==26.0.0 -markdown==3.10.2 -pytest==9.1.1 -pytest-watch==4.2.0 -pytest-cov==7.1.0 -Werkzeug!=2.2.0 diff --git a/server.js b/server.js new file mode 100644 index 00000000..f23af8d3 --- /dev/null +++ b/server.js @@ -0,0 +1,118 @@ +const express = require('express'); +const fs = require('fs-extra'); +const path = require('path'); + +const app = express(); +const PORT = process.env.PORT || 8080; + +// --------------------------------------------------------------------------- +// Security Headers Middleware +// --------------------------------------------------------------------------- +app.use((req, res, next) => { + res.setHeader('X-Frame-Options', 'SAMEORIGIN'); + res.setHeader('X-Content-Type-Options', 'nosniff'); + res.setHeader('Referrer-Policy', 'strict-origin-when-cross-origin'); + // Apply CSP header similar to app.yaml for HTML responses + const isHtml = req.path === '/' || !path.extname(req.path); + if (isHtml) { + res.setHeader('Content-Security-Policy', "default-src 'self' https:; script-src 'self' https: 'unsafe-inline'; style-src 'self' https: 'unsafe-inline'; img-src 'self' https: data:; font-src 'self' https: data:; connect-src 'self' https:; frame-ancestors 'none'"); + } + next(); +}); + +// WOFF and WOFF2 caching headers +app.use('/static/fonts', (req, res, next) => { + if (req.path.endsWith('.woff') || req.path.endsWith('.woff2')) { + res.setHeader('Cache-Control', 'public, max-age=31536000'); + } + next(); +}); + +// --------------------------------------------------------------------------- +// Static file handlers +// --------------------------------------------------------------------------- +app.get('/robots.txt', (req, res) => { + res.sendFile(path.join(__dirname, 'public/robots.txt')); +}); + +app.get('/sitemap.xml', (req, res) => { + res.sendFile(path.join(__dirname, 'public/sitemap.xml')); +}); + +app.get('/favicon.ico', (req, res) => { + res.sendFile(path.join(__dirname, 'public/static/img/favicon.ico')); +}); + +app.get('/.well-known/:file', (req, res) => { + const safeFile = path.basename(req.params.file); + res.sendFile(path.join(__dirname, 'public/static/well-known', safeFile)); +}); + +// Static assets pass-through +app.use('/static', express.static(path.join(__dirname, 'public/static'))); + + +// --------------------------------------------------------------------------- +// Local fallback static file router +// --------------------------------------------------------------------------- +app.use((req, res, next) => { + const distDir = path.resolve(__dirname, 'dist'); + + // Normalize and sanitize the request path early to break the taint chain. + // path.posix.normalize collapses redundant slashes/dots, then we reject + // anything containing traversal segments or characters outside a strict + // allowlist before the value ever reaches a file-system call. + let filePath = path.posix.normalize(req.path.replace(/^\/+/, '')); + + if (filePath.includes('..') || !/^[a-zA-Z0-9._\-/]*$/.test(filePath)) { + return next(); + } + + if (!filePath || filePath === '.') { + filePath = 'index.html'; + } else if (req.path.endsWith('/')) { + filePath = `${filePath}/index.html`; + } + + // Resolve the dist directory to its real path to guard against symlink escapes. + let realDistDir; + try { + realDistDir = fs.realpathSync(distDir); + } catch { + return next(); + } + + const fullPath = path.resolve(realDistDir, filePath); + const indexFullPath = path.resolve(realDistDir, filePath, 'index.html'); + + const isWithinDist = (candidatePath) => { + const rel = path.relative(realDistDir, candidatePath); + return !rel.startsWith('..') && !path.isAbsolute(rel); + }; + + if (fs.existsSync(fullPath) && fs.statSync(fullPath).isFile()) { + const realFullPath = fs.realpathSync(fullPath); + if (isWithinDist(realFullPath)) { + return res.sendFile(realFullPath); + } + } else if (fs.existsSync(indexFullPath) && fs.statSync(indexFullPath).isFile()) { + const realIndexFullPath = fs.realpathSync(indexFullPath); + if (isWithinDist(realIndexFullPath)) { + return res.sendFile(realIndexFullPath); + } + } + + next(); +}); + +// 404 handler +app.use((req, res) => { + res.status(404).json({ error: 'Not found', path: req.path }); +}); + +// Start Server +app.listen(PORT, () => { + console.log(`Server is running on port ${PORT}`); +}); + +module.exports = app; diff --git a/server/__init__.py b/server/__init__.py deleted file mode 100644 index 87635f69..00000000 --- a/server/__init__.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright 2015 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START app] -import os -import logging -from flask import ( - Flask, - request, - render_template as flask_render_template, - redirect, - url_for as flask_url_for, -) -from werkzeug.http import HTTP_STATUS_CODES -from flask_talisman import Talisman -from urllib.parse import urlparse, urlunparse -from .markdown import Markdown, markdown - - -from .csp import csp -from . import reports as report_util -from . import faq as faq_util -from . import timestamps as timestamps_util -from .legacy import Legacy - - -logging.basicConfig(level=logging.DEBUG) -logging.getLogger("MARKDOWN").setLevel(logging.INFO) - -ROOT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") -TEMPLATES_DIR = ROOT_DIR + "/templates" -STATIC_DIR = ROOT_DIR + "/static" - - -# Set WOFF and WOFF2 caching to return 1 year as they should never change -# Note this requires similar set up in app.yaml for Google App Engine -class HttpArchiveWebServer(Flask): - def get_send_file_max_age(self, name): - if name: - if name.lower().endswith(".woff") or name.lower().endswith(".woff2"): - return 31536000 - return Flask.get_send_file_max_age(self, name) - - -# Initialize The Server -app = HttpArchiveWebServer( - __name__, template_folder=TEMPLATES_DIR, static_folder=STATIC_DIR -) - -# register jinja2 extensions and filters - -app.jinja_options = app.jinja_options.copy() -app.jinja_env.add_extension(Markdown) -app.jinja_env.filters["markdown"] = markdown - -talisman = Talisman( - app, - content_security_policy=csp, - content_security_policy_nonce_in=["script-src"], -) -legacy_util = Legacy(faq_util) - - -@app.before_request -def redirect_www(): - """Redirect subdomain requests to bare domain.""" - urlparts = urlparse(request.url) - if ( - urlparts.netloc == "www.httparchive.org" - or urlparts.netloc == "beta.httparchive.org" - or urlparts.netloc == "legacy.httparchive.org" - ): - urlparts_list = list(urlparts) - urlparts_list[1] = "httparchive.org" - return redirect(urlunparse(urlparts_list), code=301) - return None - - -@app.after_request -def add_header(response): - # Make sure bad responses are not cached - # - # Cache good responses for 10 mins if no other Cache-Control header set - # This is used for the dynamically generated files (e.g. the HTML) - # (currently don't use unique filenames so cannot use long caches and - # some say they are overrated anyway as caches smaller than we think). - # Note this IS used by Google App Engine as dynamic content. - if "Cache-Control" not in response.headers: - if response.status_code != 200 and response.status_code != 304: - response.cache_control.no_store = True - response.cache_control.no_cache = True - response.cache_control.max_age = 0 - if response.status_code == 200 or response.status_code == 304: - response.cache_control.public = True - response.cache_control.max_age = 600 - return response - - -# Cache static resources for 10800 secs (3 hrs) with SEND_FILE_MAX_AGE_DEFAULT. -# Flask default if not set is 12 hours but we want to match app.yaml -# which is used by Google App Engine as it serves static files directly -app.config["SEND_FILE_MAX_AGE_DEFAULT"] = 10800 - - -# Overwrite the built-in method. -def render_template(template, *args, **kwargs): - - date_published = timestamps_util.get_file_date_info(template, "date_published") - date_modified = timestamps_util.get_file_date_info(template, "date_modified") - - kwargs.update(date_published=date_published, date_modified=date_modified) - return flask_render_template(template, *args, **kwargs) - - -# Overwrite the built-in method. -def url_for(endpoint, **kwargs): - # Persist the lens parameter across navigations. - lens = request.args.get("lens") - if report_util.is_valid_lens(lens): - kwargs["lens"] = lens - - # Pass through to the built-in method. - return flask_url_for(endpoint, **kwargs) - - -app.jinja_env.globals["url_for"] = url_for -app.jinja_env.globals["get_versioned_filename"] = timestamps_util.get_versioned_filename -app.jinja_env.globals["HTTP_STATUS_CODES"] = HTTP_STATUS_CODES - - -# Circular Import but this is fine because routes and errors modules are not used in here and -# this way we make app available for decorators in both modules -# For more details, check https://flask.palletsprojects.com/en/1.1.x/patterns/packages/ -import server.routes -import server.errors - -# [END app] diff --git a/server/csp.py b/server/csp.py deleted file mode 100644 index fb7dac9c..00000000 --- a/server/csp.py +++ /dev/null @@ -1,41 +0,0 @@ -csp = { - "default-src": "'self'", - "style-src": ["'self'", "'unsafe-inline'"], - "script-src": [ - "'self'", - "'strict-dynamic'", - "cdn.httparchive.org", - "*.googletagmanager.com", - "cdn.speedcurve.com", - "spdcrv.global.ssl.fastly.net", - "lux.speedcurve.com", - "'unsafe-inline'", - "reports-dev-2vzgiib6.uc.gateway.dev", - "reports-prod-2vzgiib6.uc.gateway.dev", - ], - "font-src": ["'self'"], - "connect-src": [ - "'self'", - "cdn.httparchive.org", - "discuss.httparchive.org", - "dev.to", - "cdn.rawgit.com", - "export.highcharts.com", - "www.webpagetest.org", - "*.analytics.google.com", - "*.google-analytics.com", - "*.googletagmanager.com", - "*.speedcurve.com", - "reports-dev-2vzgiib6.uc.gateway.dev", - "reports-prod-2vzgiib6.uc.gateway.dev", - ], - "img-src": [ - "'self'", - "https:", - "*.google-analytics.com", - "*.googletagmanager.com", - ], - "frame-src": ["'none'"], - "object-src": ["'none'"], - "base-uri": ["'none'"], -} diff --git a/server/dates.py b/server/dates.py deleted file mode 100644 index b667b2da..00000000 --- a/server/dates.py +++ /dev/null @@ -1,91 +0,0 @@ -import re -import logging - -from google.cloud import storage -from google.auth.exceptions import DefaultCredentialsError, RefreshError -from google.api_core.exceptions import Forbidden - -# Create/configure stream handler -sh = logging.StreamHandler() -fmt = logging.Formatter("%(name)s.py - %(levelname)s - %(message)s") -sh.setFormatter(fmt) - -# Register stream handler -logger = logging.getLogger(__name__) -logger.addHandler(sh) -mock_dates = [] - -GCS_BUCKET = "httparchive" - -# When running the site locally and unable to authenticate with GCS bypass -# loading dates from GCS. -LOAD_DATES_FROM_GCS = True -try: - gcs = storage.Client() - gcs.get_bucket(GCS_BUCKET) # pragma: no cover -except (DefaultCredentialsError, RefreshError, Forbidden): # pragma: no cover - logger.warning("Unable to authenticate to Google Cloud Storage.") - LOAD_DATES_FROM_GCS = False - - def mock_get_dates(start_year, end_year, today, month_delta=0): - """Generate mock dates to the format the site frontend expects them""" - year = start_year - months = [] - for month in range(1, ((end_year - start_year) * 12) + (month_delta + 1)): - current_month_in_year = month % 12 - if current_month_in_year == 0: - current_month_in_year = 12 - - if current_month_in_year < 10: - current_month_in_year = "0{}".format(current_month_in_year) - - months.append("{}_{}_01".format(year, current_month_in_year)) - if year < 2019: - months.append("{}_{}_15".format(year, current_month_in_year)) - - if month % 12 == 0: - year = year + 1 - - months.sort(reverse=True) - return months - - # Setup the mock dates - import datetime - - now = datetime.datetime.now(datetime.timezone.utc) - mock_dates = mock_get_dates(2010, now.year, now.day, now.month) - - -def get_dates(): # pragma: no cover - if not LOAD_DATES_FROM_GCS: - logger.warning("Google Cloud Storage disabled, using mock_get_dates()") - return mock_dates - - bucket = gcs.get_bucket(GCS_BUCKET) - iterator = bucket.list_blobs(prefix="reports/20", delimiter="/") - dates = [] - pattern = re.compile(r"([\d_]{10})") - for page in iterator.pages: - for prefix in page.prefixes: - match = pattern.search(prefix) - if match: - dates.append(match.group(0)) - dates.sort(reverse=True) - return dates - - -def get_latest_date(dates, metric_id): # pragma: no cover - if not LOAD_DATES_FROM_GCS: - logger.warning("Google Cloud Storage disabled, using mock_get_latest_date") - return mock_dates[0] - - bucket = gcs.get_bucket(GCS_BUCKET) - for date in dates: - response = bucket.get_blob("reports/%s/%s.json" % (date, metric_id)) - if response: - return date - - logger.warning( - "Could not load any dates Google Cloud Storage, using mock_get_latest_date" - ) - return mock_dates[0] diff --git a/server/errors.py b/server/errors.py deleted file mode 100644 index 78ccf63b..00000000 --- a/server/errors.py +++ /dev/null @@ -1,42 +0,0 @@ -from flask import request, redirect -import logging -from time import time -from urllib.parse import urlparse - - -from . import app, render_template, legacy_util, url_for - - -@app.errorhandler(400) -def bad_request(e): - return render_template("error/400.html", error=e), 400 - - -@app.errorhandler(404) -def page_not_found(e): - url = urlparse(request.url) - path = url.path - if legacy_util.should_redirect(path): - page = legacy_util.get_redirect_page(path) - redirect_url = url_for(page.name, **page.kwargs) - response = app.make_response(redirect(redirect_url, code=301)) - # Set a cookie that expires 5 seconds after page load, - # to ensure that it is only shown once per redirect. - # Since the redirects are permanent (301) this should only be - # shown to users the first time they hit each legacy URL. - expiration = time() + 5 - response.set_cookie("legacy_welcome", "1", expires=expiration) - return response - - return render_template("error/404.html", error=e, path=path), 404 - - -@app.errorhandler(500) -def server_error_500(e): # pragma: no cover - logging.exception("An error occurred during a request.") - return render_template("error/500.html", error=e), 500 - - -@app.errorhandler(502) -def server_error_502(e): # pragma: no cover - return render_template("error/502.html", error=e), 502 diff --git a/server/faq.py b/server/faq.py deleted file mode 100644 index c02b3e56..00000000 --- a/server/faq.py +++ /dev/null @@ -1,35 +0,0 @@ -import re -from collections import OrderedDict - -ANCHOR_BIGQUERY = None - - -def get_anchor(question): - return question.lower().replace(" ", "-").replace("?", "").replace("'", "") - - -def load_faq(): - global FAQ - global ANCHOR_BIGQUERY - global anchors - - with open("docs/faq.md") as faq_file: - FAQ = faq_file.read() - - # Any heading-level markdown (#, ##, etc) is assumed to be a question. - questions = re.findall(r"^[#]+\s*(.*)", FAQ, flags=re.MULTILINE) - anchors = OrderedDict( - (question, get_anchor(question)) for question in questions - ) - ANCHOR_BIGQUERY = find_anchor("BigQuery") - - -# Finds the first anchor whose question contains a given string. -def find_anchor(query): - questions = [question for question in anchors if query.lower() in question.lower()] - if len(questions): - return anchors.get(questions[0]) - return None # pragma: no cover - - -load_faq() diff --git a/server/legacy.py b/server/legacy.py deleted file mode 100644 index 874270b8..00000000 --- a/server/legacy.py +++ /dev/null @@ -1,26 +0,0 @@ -class Page: - def __init__(self, name, **kwargs): - self.name = name - self.kwargs = kwargs - - def set_kwargs(self, **kwargs): # pragma: no cover - self.kwargs = kwargs - - -class Legacy: - def __init__(self, faq): - self.LEGACY_PATH_MAP = { - "/index.php": Page("index"), - "/about.php": Page("about"), - "/trends.php": Page("report", report_id="state-of-the-web"), - "/interesting.php": Page( - "report", report_id="state-of-the-web", start="latest" - ), - "/downloads.php": Page("faq", _anchor=faq.ANCHOR_BIGQUERY), - } - - def should_redirect(self, path): - return path in self.LEGACY_PATH_MAP - - def get_redirect_page(self, path): - return self.LEGACY_PATH_MAP.get(path) diff --git a/server/markdown.py b/server/markdown.py deleted file mode 100644 index 3abcda7e..00000000 --- a/server/markdown.py +++ /dev/null @@ -1,57 +0,0 @@ -# Based on https://gist.github.com/mikefromit/5a6fdfecc9310712f15a872df9f41f03 - -from jinja2 import pass_environment, nodes -from jinja2.ext import Extension -import markdown as md - - -@pass_environment -def markdown(env, value): - """ - Markdown filter with support for extensions. - """ - output = value - # insert custom markdown extensions here - extensions = [ - "markdown.extensions.meta", - "markdown.extensions.attr_list", - "markdown.extensions.toc", - "markdown.extensions.def_list", - ] - - d = dict() - d["extensions"] = list() - d["extensions"].extend(extensions) - - marked = md.Markdown(**d) - - return marked.convert(output) - - -class Markdown(Extension): - """ - A wrapper around the markdown filter for syntactic sugar. - """ - - tags = set(["markdown"]) - - def parse(self, parser): # pragma: no cover - """ - Parses the statements and defers to the callback - for markdown processing. - """ - lineno = next(parser.stream).lineno - body = parser.parse_statements(["name:endmarkdown"], drop_needle=True) - - return nodes.CallBlock( - self.call_method("_render_markdown"), [], [], body - ).set_lineno(lineno) - - def _render_markdown(self, caller=None): # pragma: no cover - """ - Calls the markdown filter to transform the output. - """ - if not caller: - return "" - output = caller().strip() - return markdown(self.environment, output) diff --git a/server/reports.py b/server/reports.py deleted file mode 100644 index 62c2c161..00000000 --- a/server/reports.py +++ /dev/null @@ -1,135 +0,0 @@ -import json -from copy import deepcopy -from time import time - -from . import dates as date_util - - -class VizTypes: - HISTOGRAM = "histogram" - TIMESERIES = "timeseries" - - -# Ensure reports are updated every 3 hours. -MAX_REPORT_STALENESS = 60 * 60 * 3 - -last_report_update = 0 -latest_metric_dates = {} -latest_metric_check = {} -report_dates = [] -reports_json = {} - - -def update_reports(): - global last_report_update - - if (time() - last_report_update) < MAX_REPORT_STALENESS: - return - - global report_dates - global reports_json - - report_dates = date_util.get_dates() - - with open("config/reports.json") as reports_file: - reports_json = json.load(reports_file) - last_report_update = time() - update_reports() - - -def get_reports(): - update_reports() - - return list(map(get_report, reports_json.get("_reports"))) - - -def get_featured_reports(): - update_reports() - - return map(get_report, reports_json.get("_featured")) - - -def map_reports(report_id): # pragma: no cover - report = reports_json.get(report_id) - report["id"] = report_id - return report - - -def get_report(report_id): - report = reports_json.get(report_id) - if not report: - return None - report = deepcopy(report) - report["id"] = report_id - report["metrics"] = list(map(get_metric, report.get("metrics"))) - return report - - -def get_metric(metric_id): - metrics = reports_json.get("_metrics") - metric = deepcopy(metrics.get(metric_id)) - if not metric: # pragma: no cover - return None - metric["id"] = metric_id - return metric - - -def get_similar_reports(metric_id, current_report_id): - similar_reports = {} - reports = reports_json.get("_reports", []) - for report_id in reports: - # A report is not similar to itself. - if report_id == current_report_id: - continue - - report = reports_json.get(report_id, {}) - # A report is similar if it contains the same metric. - if "metrics" in report and metric_id in report["metrics"]: - similar_reports[report_id] = report["name"] - return similar_reports - - -def get_dates(): - return report_dates - - -def get_latest_date(metric_id): - # Check the cache before hitting GCS. - latest_date = latest_metric_dates.get(metric_id) - if latest_date: # pragma: no cover - # Only return the cached value if it's the latest date - # So reports like CrUX which come in later are checked each time - if latest_date == report_dates[0]: - return latest_date - # If it's not the latest date, but we're within our staleness time period - # then also OK to use this - if (time() - latest_metric_check[metric_id]) < MAX_REPORT_STALENESS: - return latest_date - - # If not using cached value then get the latest value - latest_date = date_util.get_latest_date(report_dates, metric_id) - latest_metric_dates[metric_id] = latest_date - latest_metric_check[metric_id] = time() - return latest_date - - -def get_lenses(): - # TODO: Consider sorting the lenses by name. - return reports_json.get("_lens", {}) - - -def get_lens(lens_id): - lenses = get_lenses() - lens = deepcopy(lenses.get(lens_id)) - if not lens: - return None - lens["id"] = lens_id - return lens - - -def is_valid_lens(lens): - lenses = get_lenses() - return lens in lenses - - -update_reports() diff --git a/server/routes.py b/server/routes.py deleted file mode 100644 index 1f2a30bb..00000000 --- a/server/routes.py +++ /dev/null @@ -1,473 +0,0 @@ -import re -from datetime import datetime - - -from flask import abort, jsonify, redirect, request, send_from_directory -from urllib.parse import urlsplit, urlunsplit, parse_qsl, urlencode - -from . import app -from . import faq as faq_util -from . import render_template -from . import reports as report_util -from . import talisman -from . import techreport as tech_report_util -from . import url_for -from .dates import get_dates - - -def safe_int(value, default=1): - """ - Safely convert a value to integer, default to 1 if conversion fails. - """ - try: - return int(value) if value else default - except (ValueError, TypeError): - return default - - -def _add_param_to_url(u: str, key: str, value: str) -> str: - """Add or update a single query param on a (possibly relative) URL string.""" - if not isinstance(u, str) or u == "": - return u # pragma: no cover - parts = urlsplit(u) # works for relative URLs like "?a=b#frag" or "#frag" - q = dict(parse_qsl(parts.query, keep_blank_values=True)) - q[key] = value # add or replace - new_query = urlencode(q, doseq=True) - return urlunsplit(parts._replace(query=new_query)) - - -def add_param_to_all_urls(obj, key: str, value: str) -> int: - """ - Recursively mutate `obj` in place, adding/updating `key=value` to all fields named "url". - """ - - if isinstance(obj, dict): - for k, v in obj.items(): - if k == "url" and isinstance(v, str): - obj[k] = _add_param_to_url(v, key, value) - else: - add_param_to_all_urls(v, key, value) - - elif isinstance(obj, list): - for item in obj: - add_param_to_all_urls(item, key, value) - - return - - -@app.route("/") -def index(): - return render_template( - "index.html", - reports=report_util.get_reports(), - featured_reports=report_util.get_featured_reports(), - faq=faq_util, - ) - - -@app.route("/about", strict_slashes=False) -def about(): - - if request.base_url[-1] == "/": - return redirect("/about"), 301 - - return render_template("about.html", reports=report_util.get_reports()) - - -@app.route("/faq", strict_slashes=False) -def faq(): - - if request.base_url[-1] == "/": - return redirect("/faq"), 301 - return render_template("faq.html", reports=report_util.get_reports(), faq=faq_util) - - -# A public JSON endpoint to get info about a given metric. -@app.route("/metric.json") -def metric(): - metric_id = request.args.get("id") - if not metric_id: - abort(jsonify(status=400, message="id parameter required")) - - metric = report_util.get_metric(metric_id) - # A metric has a histogram if it is not explicitly disabled. - has_histogram = metric and metric.get("histogram", {}).get("enabled", True) - latest = ( - report_util.get_latest_date(metric_id) if metric and has_histogram else None - ) - - return jsonify(status=200, metric=metric, latest=latest) - - -@app.route("/reports", strict_slashes=False) -def reports(): - - if request.base_url[-1] == "/": - return redirect("/reports"), 301 - - all_reports = report_util.get_reports() - - # Return as JSON if requested. - if get_format(request) == "json": - return jsonify(status=200, reports=all_reports) - - return render_template("reports.html", reports=all_reports) - - -@app.route("/reports/techreport/