Standardize environment variables on DOCSRS_

.env.sample

@@ -1,5 +1,5 @@
-CRATESFYI_PREFIX=ignored/cratesfyi-prefix
-CRATESFYI_DATABASE_URL=postgresql://cratesfyi:password@localhost:15432
+DOCSRS_PREFIX=ignored/cratesfyi-prefix
+DOCSRS_DATABASE_URL=postgresql://cratesfyi:password@localhost:15432
 DOCSRS_LOG=docs_rs,rustwide=info
 AWS_ACCESS_KEY_ID=cratesfyi
 AWS_SECRET_ACCESS_KEY=secret_key

.github/workflows/ci.yml

@@ -58,12 +58,12 @@ jobs:
         run: |
           cp .env.sample .env
           . .env
-          mkdir -p ${CRATESFYI_PREFIX}/public-html
+          mkdir -p ${DOCSRS_PREFIX}/public-html
           docker-compose up -d db s3
           # Give the database enough time to start up
           sleep 5
           # Make sure the database is actually working
-          psql "${CRATESFYI_DATABASE_URL}"
+          psql "${DOCSRS_DATABASE_URL}"
 
       - name: run tests
         shell: bash
@@ -94,12 +94,12 @@ jobs:
         run: |
           cp .env.sample .env
           . .env
-          mkdir -p ${CRATESFYI_PREFIX}/public-html
+          mkdir -p ${DOCSRS_PREFIX}/public-html
           docker-compose up -d db s3
           # Give the database enough time to start up
           sleep 5
           # Make sure the database is actually working
-          psql "${CRATESFYI_DATABASE_URL}"
+          psql "${DOCSRS_DATABASE_URL}"
 
       - name: slow tests
         env:

README.md

@@ -58,7 +58,7 @@ git clone https://github.com/rust-lang/docs.rs.git docs.rs
 cd docs.rs
 # Configure the default settings for external services
 cp .env.sample .env
-# Create the CRATESFYI_PREFIX directory
+# Create the DOCSRS_PREFIX directory
 mkdir -p ignored/cratesfyi-prefix/crates.io-index
 # Builds the docs.rs binary
 cargo build
@@ -200,7 +200,7 @@ cargo run -- build crate --local /path/to/source
 cargo run -- database add-directory <DIRECTORY> [PREFIX]
 
 # Updates repository stats for crates.
-# You need to set the CRATESFYI_GITHUB_ACCESSTOKEN
+# You need to set the DOCSRS_GITHUB_ACCESSTOKEN
 # environment variable in order to run this command.
 # Set DOCSRS_GITLAB_ACCESSTOKEN to raise the rate limit for GitLab repositories,
 # or leave it blank to fetch repositories at a slower rate.
@@ -212,7 +212,7 @@ with the `psql` command.
 
 ```sh
 . .env
-psql $CRATESFYI_DATABASE_URL
+psql $DOCSRS_DATABASE_URL
 ```
 
 The database contains a blacklist of crates that should not be built.

crates/metadata/build.rs

@@ -1,7 +1,7 @@
 fn main() {
     // Set the host target
     println!(
-        "cargo:rustc-env=DOCS_RS_METADATA_HOST_TARGET={}",
+        "cargo:rustc-env=DOCSRS_METADATA_HOST_TARGET={}",
         std::env::var("TARGET").unwrap(),
     );
     // This only needs to be rerun if the TARGET changed, in which case cargo reruns it anyway.

crates/metadata/lib.rs

@@ -50,7 +50,7 @@ use toml::Value;
 /// This is directly passed on from the Cargo [`TARGET`] variable.
 ///
 /// [`TARGET`]: https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-build-scripts
-pub const HOST_TARGET: &str = env!("DOCS_RS_METADATA_HOST_TARGET");
+pub const HOST_TARGET: &str = env!("DOCSRS_METADATA_HOST_TARGET");
 /// The targets that are built if no `targets` section is specified.
 ///
 /// Currently, this is guaranteed to have only [tier one] targets.

docker-compose.yml

@@ -16,8 +16,8 @@ services:
             - ".rustwide-docker:/opt/docsrs/rustwide"
             - "cratesio-index:/opt/docsrs/prefix/crates.io-index"
         environment:
-            CRATESFYI_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide
-            CRATESFYI_DATABASE_URL: postgresql://cratesfyi:password@db
+            DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide
+            DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@db
             DOCSRS_STORAGE_BACKEND: s3
             S3_ENDPOINT: http://s3:9000
             AWS_ACCESS_KEY_ID: cratesfyi

dockerfiles/entrypoint.sh

@@ -2,9 +2,9 @@
 
 set -euv
 
-export CRATESFYI_PREFIX=/opt/docsrs/prefix
-export DOCS_RS_DOCKER=true
-export DOCSRS_LOG=${DOCSRS_LOG-cratesfyi,rustwide=info}
+export DOCSRS_PREFIX=/opt/docsrs/prefix
+export DOCSRS_DOCKER=true
+export DOCSRS_LOG=${DOCSRS_LOG-"docs-rs,rustwide=info"}
 export PATH="$PATH:/build/target/release"
 
 # Try migrating the database multiple times if it fails
@@ -27,10 +27,10 @@ while true; do
 done
 set -e
 
-if ! [ -d "${CRATESFYI_PREFIX}/crates.io-index/.git" ]; then
-    git clone https://github.com/rust-lang/crates.io-index "${CRATESFYI_PREFIX}/crates.io-index"
+if ! [ -d "${DOCSRS_PREFIX}/crates.io-index/.git" ]; then
+    git clone https://github.com/rust-lang/crates.io-index "${DOCSRS_PREFIX}/crates.io-index"
     # Prevent new crates built before the container creation to be built
-    git --git-dir="$CRATESFYI_PREFIX/crates.io-index/.git" branch crates-index-diff_last-seen
+    git --git-dir="$DOCSRS_PREFIX/crates.io-index/.git" branch crates-index-diff_last-seen
 fi
 
 cratesfyi build update-toolchain --only-first-time

src/bin/cratesfyi.rs

@@ -388,9 +388,6 @@ enum DatabaseSubcommand {
         /// Path of file or directory
         #[structopt(name = "DIRECTORY")]
         directory: PathBuf,
-        /// Prefix of files in database
-        #[structopt(name = "PREFIX", env = "CRATESFYI_PREFIX")]
-        prefix: String,
     },
 
     /// Remove documentation from the database
@@ -440,8 +437,8 @@ impl DatabaseSubcommand {
                 )?;
             }
 
-            Self::AddDirectory { directory, prefix } => {
-                add_path_into_database(&*ctx.storage()?, &prefix, directory)
+            Self::AddDirectory { directory } => {
+                add_path_into_database(&*ctx.storage()?, &ctx.config()?.prefix, directory)
                     .context("Failed to add directory into database")?;
             }
 

src/config.rs

@@ -66,16 +66,16 @@ pub struct Config {
 
 impl Config {
     pub fn from_env() -> Result<Self, Error> {
-        let prefix: PathBuf = require_env("CRATESFYI_PREFIX")?;
+        let prefix: PathBuf = require_env("DOCSRS_PREFIX")?;
 
         Ok(Self {
             build_attempts: env("DOCSRS_BUILD_ATTEMPTS", 5)?,
 
-            prefix: prefix.clone(),
             registry_index_path: env("REGISTRY_INDEX_PATH", prefix.join("crates.io-index"))?,
             registry_url: maybe_env("REGISTRY_URL")?,
+            prefix,
 
-            database_url: require_env("CRATESFYI_DATABASE_URL")?,
+            database_url: require_env("DOCSRS_DATABASE_URL")?,
             max_pool_size: env("DOCSRS_MAX_POOL_SIZE", 90)?,
             min_pool_idle: env("DOCSRS_MIN_POOL_IDLE", 10)?,
 
@@ -90,7 +90,7 @@ impl Config {
             #[cfg(test)]
             s3_bucket_is_temporary: false,
 
-            github_accesstoken: maybe_env("CRATESFYI_GITHUB_ACCESSTOKEN")?,
+            github_accesstoken: maybe_env("DOCSRS_GITHUB_ACCESSTOKEN")?,
             github_updater_min_rate_limit: env("DOCSRS_GITHUB_UPDATER_MIN_RATE_LIMIT", 2500)?,
 
             gitlab_accesstoken: maybe_env("DOCSRS_GITLAB_ACCESSTOKEN")?,
@@ -106,12 +106,12 @@ impl Config {
 
             csp_report_only: env("DOCSRS_CSP_REPORT_ONLY", false)?,
 
-            rustwide_workspace: env("CRATESFYI_RUSTWIDE_WORKSPACE", PathBuf::from(".workspace"))?,
-            inside_docker: env("DOCS_RS_DOCKER", false)?,
-            docker_image: maybe_env("DOCS_RS_LOCAL_DOCKER_IMAGE")?
+            rustwide_workspace: env("DOCSRS_RUSTWIDE_WORKSPACE", PathBuf::from(".workspace"))?,
+            inside_docker: env("DOCSRS_DOCKER", false)?,
+            docker_image: maybe_env("DOCSRS_LOCAL_DOCKER_IMAGE")?
                 .or(maybe_env("DOCSRS_DOCKER_IMAGE")?),
-            toolchain: env("CRATESFYI_TOOLCHAIN", "nightly".to_string())?,
-            build_cpu_limit: maybe_env("DOCS_RS_BUILD_CPU_LIMIT")?,
+            toolchain: env("DOCSRS_TOOLCHAIN", "nightly".to_string())?,
+            build_cpu_limit: maybe_env("DOCSRS_BUILD_CPU_LIMIT")?,
             include_default_targets: env("DOCSRS_INCLUDE_DEFAULT_TARGETS", true)?,
             disable_memory_limit: env("DOCSRS_DISABLE_MEMORY_LIMIT", false)?,
         })

src/db/file.rs

@@ -24,10 +24,10 @@ use std::path::{Path, PathBuf};
 /// and files generated by rustdoc.
 pub fn add_path_into_database<P: AsRef<Path>>(
     storage: &Storage,
-    prefix: &str,
+    prefix: impl AsRef<Path>,
     path: P,
 ) -> Result<(Value, CompressionAlgorithms)> {
-    let (file_list, algorithms) = storage.store_all(prefix, path.as_ref())?;
+    let (file_list, algorithms) = storage.store_all(prefix.as_ref(), path.as_ref())?;
     Ok((
         file_list_to_json(file_list.into_iter().collect()),
         algorithms,

src/storage/mod.rs

@@ -152,7 +152,7 @@ impl Storage {
     // This returns (map<filename, mime type>, set<compression algorithms>).
     pub(crate) fn store_all(
         &self,
-        prefix: &str,
+        prefix: &Path,
         root_dir: &Path,
     ) -> Result<(HashMap<PathBuf, String>, HashSet<CompressionAlgorithm>), Error> {
         let mut file_paths_and_mimes = HashMap::new();
@@ -171,7 +171,7 @@ impl Storage {
             .map(|(file_path, file)| -> Result<_, Error> {
                 let alg = CompressionAlgorithm::default();
                 let content = compress(file, alg)?;
-                let bucket_path = Path::new(prefix).join(&file_path).to_slash().unwrap();
+                let bucket_path = prefix.join(&file_path).to_slash().unwrap();
 
                 let mime = detect_mime(&file_path);
                 file_paths_and_mimes.insert(file_path, mime.to_string());
@@ -464,7 +464,7 @@ mod backend_tests {
             fs::write(path, "data")?;
         }
 
-        let (stored_files, algs) = storage.store_all("prefix", dir.path())?;
+        let (stored_files, algs) = storage.store_all(Path::new("prefix"), dir.path())?;
         assert_eq!(stored_files.len(), files.len());
         for name in &files {
             let name = Path::new(name);