Implement dynamic OpenGraph image url support for crate pages

hdoordt · hdoordt · commit a0faaa4a74b4 · 2025-01-28T13:54:14.000+01:00
diff --git a/.env.sample b/.env.sample
@@ -86,3 +86,15 @@ export GH_CLIENT_SECRET=
 # Credentials for connecting to the Sentry error reporting service.
 # export SENTRY_DSN_API=
 export SENTRY_ENV_API=local
+
+# If set the server serves the frontend `index.html` for all
+# non-API requests using the Jinja template at the given path.
+# Setting this parameter requires setting
+# INDEX_HTML_TEMPLATE_PATH as well.
+export INDEX_HTML_TEMPLATE_PATH=dist/index.html
+
+# Base URL for the service from which the OpenGraph images
+# for crates are loaded. Required if
+# INDEX_HTML_TEMPLATE_PATH is set. Make sure the URL ends
+# with a `/`.
+export OG_IMAGE_BASE_URL="http://localhost:3000/og/"
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -92,7 +92,7 @@ ipnetwork = "=0.21.1"
 json-subscriber = "=0.2.4"
 krata-tokio-tar = "=0.4.2"
 lettre = { version = "=0.11.11", default-features = false, features = ["file-transport", "smtp-transport", "hostname", "builder", "tokio1", "tokio1-native-tls"] }
-minijinja = "=2.6.0"
+minijinja = {version = "=2.6.0", features = ["loader"] }
 mockall = "=0.13.1"
 native-tls = "=0.2.13"
 oauth2 = "=5.0.0"
diff --git a/app/index.html b/app/index.html
@@ -23,7 +23,7 @@
 
     <meta name="google" content="notranslate" />
 
-    <meta property="og:image" content="https://crates.io/assets/og-image.png">
+    <meta property="og:image" content="{{og_image_url}}">
     <meta name="twitter:card" content="summary_large_image">
   </head>
   <body>
diff --git a/src/config/server.rs b/src/config/server.rs
@@ -1,6 +1,7 @@
 use anyhow::{anyhow, Context};
 use ipnetwork::IpNetwork;
 use oauth2::{ClientId, ClientSecret};
+use url::Url;
 
 use crate::rate_limiter::{LimitedAction, RateLimiterConfig};
 use crate::Env;
@@ -11,11 +12,12 @@ use crate::config::cdn_log_storage::CdnLogStorageConfig;
 use crate::config::CdnLogQueueConfig;
 use crate::middleware::cargo_compat::StatusCodeConfig;
 use crate::storage::StorageConfig;
-use crates_io_env_vars::{list, list_parsed, required_var, var, var_parsed};
+use crates_io_env_vars::{list, list_parsed, required_var, required_var_parsed, var, var_parsed};
 use http::HeaderValue;
 use std::collections::{HashMap, HashSet};
 use std::convert::Infallible;
 use std::net::IpAddr;
+use std::path::PathBuf;
 use std::str::FromStr;
 use std::time::Duration;
 
@@ -71,9 +73,16 @@ pub struct Server {
     /// Should the server serve the frontend assets in the `dist` directory?
     pub serve_dist: bool,
 
-    /// Should the server serve the frontend `index.html` for all
-    /// non-API requests?
-    pub serve_html: bool,
+    /// If set the server serves the frontend `index.html` for all
+    /// non-API requests using the Jinja template at the given path.
+    /// Setting this parameter requires setting
+    /// [`Self::og_image_base_url`] as well.
+    pub index_html_template_path: Option<PathBuf>,
+
+    /// Base URL for the service from which the OpenGraph images
+    /// for crates are loaded. Required if
+    /// [`Self::index_html_template_path`] is set.
+    pub og_image_base_url: Option<Url>,
 
     pub content_security_policy: Option<HeaderValue>,
 }
@@ -171,6 +180,15 @@ impl Server {
             cdn_domain = storage.cdn_prefix.as_ref().map(|cdn_prefix| format!("https://{cdn_prefix}")).unwrap_or_default()
         );
 
+        let index_html_template_path = var_parsed("INDEX_HTML_TEMPLATE_PATH")?;
+        let og_image_base_url = match index_html_template_path {
+            Some(_) => Some(
+                required_var_parsed("OG_IMAGE_BASE_URL")
+                    .context("OG_IMAGE_BASE_URL must be set when using INDEX_HTML_TEMPLATE_PATH")?,
+            ),
+            None => None,
+        };
+
         Ok(Server {
             db: DatabasePools::full_from_environment(&base)?,
             storage,
@@ -214,7 +232,8 @@ impl Server {
             cargo_compat_status_code_config: var_parsed("CARGO_COMPAT_STATUS_CODES")?
                 .unwrap_or(StatusCodeConfig::AdjustAll),
             serve_dist: true,
-            serve_html: true,
+            index_html_template_path,
+            og_image_base_url,
             content_security_policy: Some(content_security_policy.parse()?),
         })
     }
diff --git a/src/middleware.rs b/src/middleware.rs
@@ -77,9 +77,10 @@ pub fn apply_axum_middleware(state: AppState, router: Router<()>) -> Router {
         .layer(conditional_layer(config.serve_dist, || {
             from_fn(static_or_continue::serve_dist)
         }))
-        .layer(conditional_layer(config.serve_html, || {
-            from_fn_with_state(state.clone(), ember_html::serve_html)
-        }))
+        .layer(conditional_layer(
+            config.index_html_template_path.is_some(),
+            || from_fn_with_state(state.clone(), ember_html::serve_html),
+        ))
         .layer(AddExtensionLayer::new(state.clone()));
 
     router
diff --git a/src/middleware/ember_html.rs b/src/middleware/ember_html.rs
@@ -7,16 +7,45 @@
 //! For now, there is an additional check to see if the `Accept` header contains "html". This is
 //! likely to be removed in the future.
 
+use std::borrow::Cow;
+use std::path::Path;
+use std::sync::OnceLock;
+
 use axum::extract::Request;
 use axum::middleware::Next;
 use axum::response::{IntoResponse, Response};
-use http::{header, StatusCode};
-use tower::ServiceExt;
-use tower_http::services::ServeFile;
+use futures_util::future::{BoxFuture, Shared};
+use futures_util::FutureExt;
+use http::{header, HeaderMap, HeaderValue, Method, StatusCode};
+use minijinja::{context, Environment};
 
-pub async fn serve_html(request: Request, next: Next) -> Response {
-    let path = &request.uri().path();
+use crate::app::AppState;
+
+const OG_IMAGE_FALLBACK_URL: &str = "https://crates.io/assets/og-image.png";
+const INDEX_TEMPLATE_NAME: &str = "index_html";
+const PATH_PREFIX_CRATES: &str = "/crates/";
+
+type TemplateEnvFut = Shared<BoxFuture<'static, minijinja::Environment<'static>>>;
+
+/// Initialize [`minijinja::Environment`] given the path to the index.html file. This should
+/// only be done once as it will load said file from persistent storage.
+async fn init_template_env(
+    index_html_template_path: impl AsRef<Path>,
+) -> minijinja::Environment<'static> {
+    let template_j2 = tokio::fs::read_to_string(index_html_template_path.as_ref())
+        .await
+        .expect("Error loading index.html template. Is the frontend package built yet?");
+
+    let mut env = Environment::empty();
+    env.add_template_owned(INDEX_TEMPLATE_NAME, template_j2)
+        .expect("Error loading template");
+    env
+}
+
+pub async fn serve_html(state: AppState, request: Request, next: Next) -> Response {
+    static TEMPLATE_ENV: OnceLock<TemplateEnvFut> = OnceLock::new();
 
+    let path = &request.uri().path();
     // The "/git/" prefix is only used in development (when within a docker container)
     if path.starts_with("/api/") || path.starts_with("/git/") {
         next.run(request).await
@@ -26,12 +55,58 @@ pub async fn serve_html(request: Request, next: Next) -> Response {
         .iter()
         .any(|val| val.to_str().unwrap_or_default().contains("html"))
     {
+        if !matches!(*request.method(), Method::HEAD | Method::GET) {
+            let headers =
+                HeaderMap::from_iter([(header::ALLOW, HeaderValue::from_static("GET,HEAD"))]);
+            return (StatusCode::METHOD_NOT_ALLOWED, headers).into_response();
+        }
+
+        // Come up with an Open Graph image URL. In case a crate page is requested,
+        // we use the crate's name and the OG image base URL from config to
+        // generate one, otherwise we use the fallback image.
+        let og_image_url: Cow<'_, _> = if let Some(suffix) = path.strip_prefix(PATH_PREFIX_CRATES) {
+            let len = suffix.find('/').unwrap_or(suffix.len());
+            let krate = &suffix[..len];
+
+            // `state.config.og_image_base_url` will always be `Some` as that's required
+            // if `state.config.index_html_template_path` is `Some`, and otherwise this
+            // middleware won't be executed; see `crate::middleware::apply_axum_middleware`.
+            if let Ok(og_img_url) = state.config.og_image_base_url.as_ref().unwrap().join(krate) {
+                Cow::from(og_img_url.to_string())
+            } else {
+                OG_IMAGE_FALLBACK_URL.into()
+            }
+        } else {
+            OG_IMAGE_FALLBACK_URL.into()
+        };
+
+        // `OnceLock::get_or_init` blocks as long as its intializer is running in another thread.
+        // Note that this won't take long, as the constructed Futures are not awaited
+        // during initialization.
+        let template_env = TEMPLATE_ENV.get_or_init(|| {
+            // At this point we can safely assume `state.config.index_html_template_path` is `Some`,
+            // as this middleware won't be executed otherwise; see `crate::middleware::apply_axum_middleware`.
+            init_template_env(state.config.index_html_template_path.clone().unwrap())
+                .boxed()
+                .shared()
+        });
+
+        // TODO use moka caching here with og_image_url as key and the rendered html as value
+
+        // Render the HTML given the OG image URL
+        let env = template_env.clone().await;
+        let html = env
+            .get_template(INDEX_TEMPLATE_NAME)
+            .unwrap()
+            .render(context! { og_image_url})
+            .expect("Error rendering index");
+
         // Serve static Ember page to bootstrap the frontend
-        ServeFile::new("dist/index.html")
-            .oneshot(request)
-            .await
-            .map(|response| response.map(axum::body::Body::new))
-            .unwrap_or_else(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())
+        Response::builder()
+            .header(header::CONTENT_TYPE, "text/html")
+            .header(header::CONTENT_LENGTH, html.len())
+            .body(axum::body::Body::new(html))
+            .unwrap()
     } else {
         // Return a 404 to crawlers that don't send `Accept: text/hml`.
         // This is to preserve legacy behavior and will likely change.
diff --git a/src/middleware/static_or_continue.rs b/src/middleware/static_or_continue.rs
@@ -18,7 +18,10 @@ pub async fn serve_dist(request: Request, next: Next) -> Response {
 }
 
 async fn serve<P: AsRef<Path>>(path: P, request: Request, next: Next) -> Response {
-    if request.method() == Method::GET || request.method() == Method::HEAD {
+    // index.html is a Jinja template, which is to be rendered by `ember_html::serve_html`.
+    if matches!(*request.method(), Method::GET | Method::HEAD)
+        && !matches!(request.uri().path().as_bytes(), b"/" | b"/index.html")
+    {
         let mut static_req = Request::new(());
         *static_req.method_mut() = request.method().clone();
         *static_req.uri_mut() = request.uri().clone();
diff --git a/src/tests/util/test_app.rs b/src/tests/util/test_app.rs
@@ -481,7 +481,8 @@ fn simple_config() -> config::Server {
 
         // The frontend code is not needed for the backend tests.
         serve_dist: false,
-        serve_html: false,
+        index_html_template_path: None,
+        og_image_base_url: None,
         content_security_policy: None,
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -481,7 +481,8 @@ fn simple_config() -> config::Server {`
`481`	`481`
`482`	`482`	`// The frontend code is not needed for the backend tests.`
`483`	`483`	`serve_dist: false,`
`484`		`- serve_html: false,`
	`484`	`+ index_html_template_path: None,`
	`485`	`+ og_image_base_url: None,`
`485`	`486`	`content_security_policy: None,`
`486`	`487`	`}`
`487`	`488`	`}`