Skip to content

Refer to top-level persistence namespaces as primary_namespace #2612

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Sep 28, 2023
Merged

Refer to top-level persistence namespaces as primary_namespace #2612

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
910a00e
Use `Result<_, io::Error>` over `io::Result<_>`
TheBlueMatt Sep 27, 2023
18ef80f
Refer to top-level persistence namespaces as `primary_namespace`
TheBlueMatt Sep 28, 2023
6765889
Rename the persistence `sub_namespace` to `secondary_namespace`
TheBlueMatt Sep 28, 2023
1cb8103
Update storage constants to new PRIMARY/SECONDARY namespace terms
TheBlueMatt Sep 28, 2023
47e1148
Update remaining references to primary/secondary namespaces
TheBlueMatt Sep 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 22 additions & 19 deletions lightning-background-processor/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -506,10 +506,10 @@ use core::task;
/// # use lightning_background_processor::{process_events_async, GossipSync};
/// # struct MyStore {}
/// # impl lightning::util::persist::KVStore for MyStore {
/// # fn read(&self, namespace: &str, sub_namespace: &str, key: &str) -> io::Result<Vec<u8>> { Ok(Vec::new()) }
/// # fn write(&self, namespace: &str, sub_namespace: &str, key: &str, buf: &[u8]) -> io::Result<()> { Ok(()) }
/// # fn remove(&self, namespace: &str, sub_namespace: &str, key: &str, lazy: bool) -> io::Result<()> { Ok(()) }
/// # fn list(&self, namespace: &str, sub_namespace: &str) -> io::Result<Vec<String>> { Ok(Vec::new()) }
/// # fn read(&self, primary_namespace: &str, secondary_namespace: &str, key: &str) -> io::Result<Vec<u8>> { Ok(Vec::new()) }
/// # fn write(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: &[u8]) -> io::Result<()> { Ok(()) }
/// # fn remove(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool) -> io::Result<()> { Ok(()) }
/// # fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result<Vec<String>> { Ok(Vec::new()) }
/// # }
/// # struct MyEventHandler {}
/// # impl MyEventHandler {
Expand Down Expand Up @@ -868,7 +868,10 @@ mod tests {
use lightning::util::config::UserConfig;
use lightning::util::ser::Writeable;
use lightning::util::test_utils;
use lightning::util::persist::{KVStore, CHANNEL_MANAGER_PERSISTENCE_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_SUB_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_KEY, NETWORK_GRAPH_PERSISTENCE_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_SUB_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_KEY, SCORER_PERSISTENCE_NAMESPACE, SCORER_PERSISTENCE_SUB_NAMESPACE, SCORER_PERSISTENCE_KEY};
use lightning::util::persist::{KVStore,
CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_KEY,
NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_KEY,
SCORER_PERSISTENCE_PRIMARY_NAMESPACE, SCORER_PERSISTENCE_SECONDARY_NAMESPACE, SCORER_PERSISTENCE_KEY};
use lightning_persister::fs_store::FilesystemStore;
use std::collections::VecDeque;
use std::{fs, env};
Expand Down Expand Up @@ -983,22 +986,22 @@ mod tests {
}

impl KVStore for Persister {
fn read(&self, namespace: &str, sub_namespace: &str, key: &str) -> lightning::io::Result<Vec<u8>> {
self.kv_store.read(namespace, sub_namespace, key)
fn read(&self, primary_namespace: &str, secondary_namespace: &str, key: &str) -> lightning::io::Result<Vec<u8>> {
self.kv_store.read(primary_namespace, secondary_namespace, key)
}

fn write(&self, namespace: &str, sub_namespace: &str, key: &str, buf: &[u8]) -> lightning::io::Result<()> {
if namespace == CHANNEL_MANAGER_PERSISTENCE_NAMESPACE &&
sub_namespace == CHANNEL_MANAGER_PERSISTENCE_SUB_NAMESPACE &&
fn write(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: &[u8]) -> lightning::io::Result<()> {
if primary_namespace == CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE &&
secondary_namespace == CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE &&
key == CHANNEL_MANAGER_PERSISTENCE_KEY
{
if let Some((error, message)) = self.manager_error {
return Err(std::io::Error::new(error, message))
}
}

if namespace == NETWORK_GRAPH_PERSISTENCE_NAMESPACE &&
sub_namespace == NETWORK_GRAPH_PERSISTENCE_SUB_NAMESPACE &&
if primary_namespace == NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE &&
secondary_namespace == NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE &&
key == NETWORK_GRAPH_PERSISTENCE_KEY
{
if let Some(sender) = &self.graph_persistence_notifier {
Expand All @@ -1013,24 +1016,24 @@ mod tests {
}
}

if namespace == SCORER_PERSISTENCE_NAMESPACE &&
sub_namespace == SCORER_PERSISTENCE_SUB_NAMESPACE &&
if primary_namespace == SCORER_PERSISTENCE_PRIMARY_NAMESPACE &&
secondary_namespace == SCORER_PERSISTENCE_SECONDARY_NAMESPACE &&
key == SCORER_PERSISTENCE_KEY
{
if let Some((error, message)) = self.scorer_error {
return Err(std::io::Error::new(error, message))
}
}

self.kv_store.write(namespace, sub_namespace, key, buf)
self.kv_store.write(primary_namespace, secondary_namespace, key, buf)
}

fn remove(&self, namespace: &str, sub_namespace: &str, key: &str, lazy: bool) -> lightning::io::Result<()> {
self.kv_store.remove(namespace, sub_namespace, key, lazy)
fn remove(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool) -> lightning::io::Result<()> {
self.kv_store.remove(primary_namespace, secondary_namespace, key, lazy)
}

fn list(&self, namespace: &str, sub_namespace: &str) -> lightning::io::Result<Vec<String>> {
self.kv_store.list(namespace, sub_namespace)
fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> lightning::io::Result<Vec<String>> {
self.kv_store.list(primary_namespace, secondary_namespace)
}
}

Expand Down
46 changes: 23 additions & 23 deletions lightning-persister/src/fs_store.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ impl FilesystemStore {
}
}

fn get_dest_dir_path(&self, namespace: &str, sub_namespace: &str) -> std::io::Result<PathBuf> {
fn get_dest_dir_path(&self, primary_namespace: &str, secondary_namespace: &str) -> std::io::Result<PathBuf> {
let mut dest_dir_path = {
#[cfg(target_os = "windows")]
{
Expand All @@ -81,20 +81,20 @@ impl FilesystemStore {
}
};

dest_dir_path.push(namespace);
if !sub_namespace.is_empty() {
dest_dir_path.push(sub_namespace);
dest_dir_path.push(primary_namespace);
if !secondary_namespace.is_empty() {
dest_dir_path.push(secondary_namespace);
}

Ok(dest_dir_path)
}
}

impl KVStore for FilesystemStore {
fn read(&self, namespace: &str, sub_namespace: &str, key: &str) -> std::io::Result<Vec<u8>> {
check_namespace_key_validity(namespace, sub_namespace, Some(key), "read")?;
fn read(&self, primary_namespace: &str, secondary_namespace: &str, key: &str) -> std::io::Result<Vec<u8>> {
check_namespace_key_validity(primary_namespace, secondary_namespace, Some(key), "read")?;

let mut dest_file_path = self.get_dest_dir_path(namespace, sub_namespace)?;
let mut dest_file_path = self.get_dest_dir_path(primary_namespace, secondary_namespace)?;
dest_file_path.push(key);

let mut buf = Vec::new();
Expand All @@ -114,10 +114,10 @@ impl KVStore for FilesystemStore {
Ok(buf)
}

fn write(&self, namespace: &str, sub_namespace: &str, key: &str, buf: &[u8]) -> std::io::Result<()> {
check_namespace_key_validity(namespace, sub_namespace, Some(key), "write")?;
fn write(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: &[u8]) -> std::io::Result<()> {
check_namespace_key_validity(primary_namespace, secondary_namespace, Some(key), "write")?;

let mut dest_file_path = self.get_dest_dir_path(namespace, sub_namespace)?;
let mut dest_file_path = self.get_dest_dir_path(primary_namespace, secondary_namespace)?;
dest_file_path.push(key);

let parent_directory = dest_file_path
Expand Down Expand Up @@ -201,10 +201,10 @@ impl KVStore for FilesystemStore {
res
}

fn remove(&self, namespace: &str, sub_namespace: &str, key: &str, lazy: bool) -> std::io::Result<()> {
check_namespace_key_validity(namespace, sub_namespace, Some(key), "remove")?;
fn remove(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool) -> std::io::Result<()> {
check_namespace_key_validity(primary_namespace, secondary_namespace, Some(key), "remove")?;

let mut dest_file_path = self.get_dest_dir_path(namespace, sub_namespace)?;
let mut dest_file_path = self.get_dest_dir_path(primary_namespace, secondary_namespace)?;
dest_file_path.push(key);

if !dest_file_path.is_file() {
Expand Down Expand Up @@ -290,10 +290,10 @@ impl KVStore for FilesystemStore {
Ok(())
}

fn list(&self, namespace: &str, sub_namespace: &str) -> std::io::Result<Vec<String>> {
check_namespace_key_validity(namespace, sub_namespace, None, "list")?;
fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> std::io::Result<Vec<String>> {
check_namespace_key_validity(primary_namespace, secondary_namespace, None, "list")?;

let prefixed_dest = self.get_dest_dir_path(namespace, sub_namespace)?;
let prefixed_dest = self.get_dest_dir_path(primary_namespace, secondary_namespace)?;
let mut keys = Vec::new();

if !Path::new(&prefixed_dest).exists() {
Expand All @@ -320,17 +320,17 @@ impl KVStore for FilesystemStore {

let metadata = p.metadata()?;

// We allow the presence of directories in the empty namespace and just skip them.
// We allow the presence of directories in the empty primary namespace and just skip them.
if metadata.is_dir() {
continue;
}

// If we otherwise don't find a file at the given path something went wrong.
if !metadata.is_file() {
debug_assert!(false, "Failed to list keys of {}/{}: file couldn't be accessed.",
PrintableString(namespace), PrintableString(sub_namespace));
PrintableString(primary_namespace), PrintableString(secondary_namespace));
let msg = format!("Failed to list keys of {}/{}: file couldn't be accessed.",
PrintableString(namespace), PrintableString(sub_namespace));
PrintableString(primary_namespace), PrintableString(secondary_namespace));
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
}

Expand All @@ -342,17 +342,17 @@ impl KVStore for FilesystemStore {
}
} else {
debug_assert!(false, "Failed to list keys of {}/{}: file path is not valid UTF-8",
PrintableString(namespace), PrintableString(sub_namespace));
PrintableString(primary_namespace), PrintableString(secondary_namespace));
let msg = format!("Failed to list keys of {}/{}: file path is not valid UTF-8",
PrintableString(namespace), PrintableString(sub_namespace));
PrintableString(primary_namespace), PrintableString(secondary_namespace));
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
}
}
Err(e) => {
debug_assert!(false, "Failed to list keys of {}/{}: {}",
PrintableString(namespace), PrintableString(sub_namespace), e);
PrintableString(primary_namespace), PrintableString(secondary_namespace), e);
let msg = format!("Failed to list keys of {}/{}: {}",
PrintableString(namespace), PrintableString(sub_namespace), e);
PrintableString(primary_namespace), PrintableString(secondary_namespace), e);
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
}
}
Expand Down
25 changes: 13 additions & 12 deletions lightning-persister/src/test_utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,34 +12,35 @@ use std::panic::RefUnwindSafe;
pub(crate) fn do_read_write_remove_list_persist<K: KVStore + RefUnwindSafe>(kv_store: &K) {
let data = [42u8; 32];

let namespace = "testspace";
let sub_namespace = "testsubspace";
let primary_namespace = "testspace";
let secondary_namespace = "testsubspace";
let key = "testkey";

// Test the basic KVStore operations.
kv_store.write(namespace, sub_namespace, key, &data).unwrap();
kv_store.write(primary_namespace, secondary_namespace, key, &data).unwrap();

// Test empty namespace/sub_namespace is allowed, but not empty namespace and non-empty
// sub-namespace, and not empty key.
// Test empty primary_namespace/secondary_namespace is allowed, but not empty primary_namespace
// and non-empty secondary_namespace, and not empty key.
kv_store.write("", "", key, &data).unwrap();
let res = std::panic::catch_unwind(|| kv_store.write("", sub_namespace, key, &data));
let res = std::panic::catch_unwind(|| kv_store.write("", secondary_namespace, key, &data));
assert!(res.is_err());
let res = std::panic::catch_unwind(|| kv_store.write(namespace, sub_namespace, "", &data));
let res = std::panic::catch_unwind(|| kv_store.write(primary_namespace, secondary_namespace, "", &data));
assert!(res.is_err());

let listed_keys = kv_store.list(namespace, sub_namespace).unwrap();
let listed_keys = kv_store.list(primary_namespace, secondary_namespace).unwrap();
assert_eq!(listed_keys.len(), 1);
assert_eq!(listed_keys[0], key);

let read_data = kv_store.read(namespace, sub_namespace, key).unwrap();
let read_data = kv_store.read(primary_namespace, secondary_namespace, key).unwrap();
assert_eq!(data, &*read_data);

kv_store.remove(namespace, sub_namespace, key, false).unwrap();
kv_store.remove(primary_namespace, secondary_namespace, key, false).unwrap();

let listed_keys = kv_store.list(namespace, sub_namespace).unwrap();
let listed_keys = kv_store.list(primary_namespace, secondary_namespace).unwrap();
assert_eq!(listed_keys.len(), 0);

// Ensure we have no issue operating with namespace/sub_namespace/key being KVSTORE_NAMESPACE_KEY_MAX_LEN
// Ensure we have no issue operating with primary_namespace/secondary_namespace/key being
// KVSTORE_NAMESPACE_KEY_MAX_LEN
let max_chars: String = std::iter::repeat('A').take(KVSTORE_NAMESPACE_KEY_MAX_LEN).collect();
kv_store.write(&max_chars, &max_chars, &max_chars, &data).unwrap();

Expand Down
48 changes: 25 additions & 23 deletions lightning-persister/src/utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,51 +6,53 @@ pub(crate) fn is_valid_kvstore_str(key: &str) -> bool {
key.len() <= KVSTORE_NAMESPACE_KEY_MAX_LEN && key.chars().all(|c| KVSTORE_NAMESPACE_KEY_ALPHABET.contains(c))
}

pub(crate) fn check_namespace_key_validity(namespace: &str, sub_namespace: &str, key: Option<&str>, operation: &str) -> Result<(), std::io::Error> {
pub(crate) fn check_namespace_key_validity(
primary_namespace: &str, secondary_namespace: &str, key: Option<&str>, operation: &str)
-> Result<(), std::io::Error> {
if let Some(key) = key {
if key.is_empty() {
debug_assert!(false, "Failed to {} {}/{}/{}: key may not be empty.", operation,
PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
let msg = format!("Failed to {} {}/{}/{}: key may not be empty.", operation,
PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
}

if namespace.is_empty() && !sub_namespace.is_empty() {
if primary_namespace.is_empty() && !secondary_namespace.is_empty() {
debug_assert!(false,
"Failed to {} {}/{}/{}: namespace may not be empty if a non-empty sub-namespace is given.",
"Failed to {} {}/{}/{}: primary namespace may not be empty if a non-empty secondary namespace is given.",
operation,
PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
let msg = format!(
"Failed to {} {}/{}/{}: namespace may not be empty if a non-empty sub-namespace is given.", operation,
PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
"Failed to {} {}/{}/{}: primary namespace may not be empty if a non-empty secondary namespace is given.", operation,
PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
}

if !is_valid_kvstore_str(namespace) || !is_valid_kvstore_str(sub_namespace) || !is_valid_kvstore_str(key) {
debug_assert!(false, "Failed to {} {}/{}/{}: namespace, sub-namespace, and key must be valid.",
if !is_valid_kvstore_str(primary_namespace) || !is_valid_kvstore_str(secondary_namespace) || !is_valid_kvstore_str(key) {
debug_assert!(false, "Failed to {} {}/{}/{}: primary namespace, secondary namespace, and key must be valid.",
operation,
PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
let msg = format!("Failed to {} {}/{}/{}: namespace, sub-namespace, and key must be valid.",
PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
let msg = format!("Failed to {} {}/{}/{}: primary namespace, secondary namespace, and key must be valid.",
operation,
PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
}
} else {
if namespace.is_empty() && !sub_namespace.is_empty() {
if primary_namespace.is_empty() && !secondary_namespace.is_empty() {
debug_assert!(false,
"Failed to {} {}/{}: namespace may not be empty if a non-empty sub-namespace is given.",
operation, PrintableString(namespace), PrintableString(sub_namespace));
"Failed to {} {}/{}: primary namespace may not be empty if a non-empty secondary namespace is given.",
operation, PrintableString(primary_namespace), PrintableString(secondary_namespace));
let msg = format!(
"Failed to {} {}/{}: namespace may not be empty if a non-empty sub-namespace is given.",
operation, PrintableString(namespace), PrintableString(sub_namespace));
"Failed to {} {}/{}: primary namespace may not be empty if a non-empty secondary namespace is given.",
operation, PrintableString(primary_namespace), PrintableString(secondary_namespace));
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
}
if !is_valid_kvstore_str(namespace) || !is_valid_kvstore_str(sub_namespace) {
debug_assert!(false, "Failed to {} {}/{}: namespace and sub-namespace must be valid.",
operation, PrintableString(namespace), PrintableString(sub_namespace));
let msg = format!("Failed to {} {}/{}: namespace and sub-namespace must be valid.",
operation, PrintableString(namespace), PrintableString(sub_namespace));
if !is_valid_kvstore_str(primary_namespace) || !is_valid_kvstore_str(secondary_namespace) {
debug_assert!(false, "Failed to {} {}/{}: primary namespace and secondary namespace must be valid.",
operation, PrintableString(primary_namespace), PrintableString(secondary_namespace));
let msg = format!("Failed to {} {}/{}: primary namespace and secondary namespace must be valid.",
operation, PrintableString(primary_namespace), PrintableString(secondary_namespace));
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
}
}
Expand Down
Loading