Raise backoffs on cron & scheduled jobs (#36344)

GitOrigin-RevId: 55f95b47f2fb05a2ef321c0ee723773ba0fb5602

Author: goffrie

crates/application/src/cron_jobs/mod.rs

@@ -82,8 +82,8 @@ use crate::{
 
 mod metrics;
 
-const INITIAL_BACKOFF: Duration = Duration::from_millis(10);
-const MAX_BACKOFF: Duration = Duration::from_secs(5);
+const INITIAL_BACKOFF: Duration = Duration::from_millis(500);
+const MAX_BACKOFF: Duration = Duration::from_secs(15);
 
 // Truncate result and log lines for cron job logs since they are only
 // used for the dashboard

crates/common/src/knobs.rs

@@ -279,7 +279,7 @@ pub static SCHEDULED_JOB_EXECUTION_PARALLELISM: LazyLock<usize> =
 
 /// Initial backoff in milliseconds on a system error from a scheduled job.
 pub static SCHEDULED_JOB_INITIAL_BACKOFF: LazyLock<Duration> =
-    LazyLock::new(|| Duration::from_millis(env_config("SCHEDULED_JOB_INITIAL_BACKOFF_MS", 10)));
+    LazyLock::new(|| Duration::from_millis(env_config("SCHEDULED_JOB_INITIAL_BACKOFF_MS", 500)));
 
 /// Max backoff in seconds on a system error from a scheduled job.
 /// Scheduled jobs can hit many OCCs, so we may need to slow them down if they