move pydantic_core -> python/pydantic_core (#705)

Author: davidhewitt

.cargo/config.toml

@@ -1,19 +1,15 @@
 [build]
-rustflags = [
-  "-A", "incomplete_features",
-]
+rustflags = []
 
 # see https://pyo3.rs/main/building_and_distribution.html#macos
 [target.x86_64-apple-darwin]
 rustflags = [
   "-C", "link-arg=-undefined",
   "-C", "link-arg=dynamic_lookup",
-  "-A", "incomplete_features",
 ]
 
 [target.aarch64-apple-darwin]
 rustflags = [
   "-C", "link-arg=-undefined",
   "-C", "link-arg=dynamic_lookup",
-  "-A", "incomplete_features",
 ]

.github/workflows/ci.yml

@@ -53,7 +53,7 @@ jobs:
       - run: ls -lha
       - run: coverage xml
 
-      - run: coverage-prepare lcov pydantic_core/*.so
+      - run: coverage-prepare lcov python/pydantic_core/*.so
 
       - uses: codecov/codecov-action@v3
 
@@ -151,8 +151,7 @@ jobs:
           python-version: '3.11'
 
       - run: pip install -r tests/requirements.txt
-      - run: 'pip install "maturin>=1,<2" typing_extensions'
-      - run: make build-dev
+      - run: pip install -e . --config-settings=build-args='--profile dev'
 
       - run: pip freeze
       - run: pytest
@@ -464,8 +463,6 @@ jobs:
             fi
           run: |
             set -x
-            # this is required so that pytest uses the installed pydantic-core package
-            rm -r pydantic_core
             # typing-extensions isn't automatically installed because of `--no-index --no-deps`
             python3 -m venv venv
             source venv/bin/activate
@@ -499,7 +496,6 @@ jobs:
           name: pypi_files
           path: dist
 
-      - run: rm -r pydantic_core
       - run: pip install typing-extensions
       - run: pip install -r tests/requirements.txt
       - run: pip install pydantic-core --no-index --no-deps --find-links dist --force-reinstall

.github/workflows/codspeed.yml

@@ -47,17 +47,13 @@ jobs:
         uses: Swatinem/rust-cache@v2
 
       - name: Install pydantic-core
-        # build-prod does not include MiMalloc because it bypasses Maturin
+        # --no-default-features to avoid using mimalloc
         run: |
-          pip install -e .
-          make build-prod
-          python -c 'import pydantic_core'
+          pip install -e . --config-settings=build-args='--no-default-features --verbose' -v
+          python -c 'import pydantic_core; assert pydantic_core._pydantic_core.__pydantic_core_default_allocator__'
         env:
           CONST_RANDOM_SEED: 0 # Fix the compile time RNG seed
 
-      # this is required so that pytest uses the installed package
-      - run: rm tests/__init__.py
-
       - name: Run CodSpeed benchmarks
         uses: CodSpeedHQ/action@v1
         with:

.gitignore

@@ -34,5 +34,5 @@ node_modules/
 /*.profdata
 /*.profraw
 /foobar.py
-/pydantic_core/*.so
+/python/pydantic_core/*.so
 /src/self_schema.py

Cargo.toml

@@ -26,14 +26,15 @@ include = [
 ]
 
 [dependencies]
-pyo3 = "0.19.0"
+pyo3 = { version = "0.19.0", features = ["generate-import-lib", "num-bigint"] }
 regex = "1.6.0"
 strum = { version = "0.25.0", features = ["derive"] }
 strum_macros = "0.24.3"
 serde_json = {version = "1.0.87", features = ["preserve_order"]}
 enum_dispatch = "0.3.8"
 serde = "1.0.147"
-mimalloc = { version = "0.1.30", default-features = false, optional = true }
+# disabled for benchmarks since it makes microbenchmark performance more flakey
+mimalloc = { version = "0.1.30", optional = true, default-features = false, features = ["local_dynamic_tls"] }
 speedate = "0.9.0"
 ahash = "0.8.0"
 url = "2.3.1"
@@ -49,11 +50,7 @@ crate-type = ["cdylib", "rlib"]
 [features]
 # must be enabled when building with `cargo build`, maturin enables this automatically
 extension-module = ["pyo3/extension-module"]
-# required for cargo bench
-auto-initialize = ["pyo3/auto-initialize"]
-# disabled for benchmarks since it makes microbenchmark performance more flakey
-mimalloc-allocator = ["mimalloc", "mimalloc/local_dynamic_tls"]
-default = ["pyo3/generate-import-lib", "pyo3/num-bigint"]
+default = ["mimalloc"]
 
 [profile.release]
 lto = "fat"

Makefile

@@ -1,6 +1,6 @@
 .DEFAULT_GOAL := all
-black = black pydantic_core tests generate_self_schema.py wasm-preview/run_tests.py
-ruff = ruff pydantic_core tests generate_self_schema.py wasm-preview/run_tests.py
+black = black python/pydantic_core tests generate_self_schema.py wasm-preview/run_tests.py
+ruff = ruff python/pydantic_core tests generate_self_schema.py wasm-preview/run_tests.py
 
 .PHONY: install
 install:
@@ -17,28 +17,18 @@ install-rust-coverage:
 
 .PHONY: build-dev
 build-dev:
-	@rm -f pydantic_core/*.so
-	cargo build --features extension-module
-	@rm -f target/debug/lib_pydantic_core.d
-	@rm -f target/debug/lib_pydantic_core.rlib
-	@mv target/debug/lib_pydantic_core.* pydantic_core/_pydantic_core.so
+	@rm -f python/pydantic_core/*.so
+	maturin develop
 
 .PHONY: build-prod
 build-prod:
-	@rm -f pydantic_core/*.so
-	cargo build --release --features extension-module
-	@rm -f target/release/lib_pydantic_core.d
-	@rm -f target/release/lib_pydantic_core.rlib
-	@mv target/release/lib_pydantic_core.* pydantic_core/_pydantic_core.so
+	@rm -f python/pydantic_core/*.so
+	maturin develop --release
 
 .PHONY: build-coverage
 build-coverage:
-	pip uninstall -y pydantic_core
-	rm -f pydantic_core/*.so
-	RUSTFLAGS='-C instrument-coverage -A incomplete_features' cargo build --features extension-module
-	@rm -f target/debug/lib_pydantic_core.d
-	@rm -f target/debug/lib_pydantic_core.rlib
-	mv target/debug/lib_pydantic_core.* pydantic_core/_pydantic_core.so
+	rm -f python/pydantic_core/*.so
+	maturin develop -- -C instrument-coverage
 
 .PHONY: build-wasm
 build-wasm:
@@ -56,7 +46,7 @@ format:
 lint-python:
 	$(ruff)
 	$(black) --check --diff
-	griffe dump -f -d google -LWARNING -o/dev/null pydantic_core
+	griffe dump -f -d google -LWARNING -o/dev/null python/pydantic_core
 
 .PHONY: lint-rust
 lint-rust:
@@ -110,7 +100,7 @@ testcov: build-coverage
 	coverage run -m pytest
 	coverage report
 	coverage html -d htmlcov/python
-	coverage-prepare html pydantic_core/*.so
+	coverage-prepare html python/pydantic_core/*.so
 
 .PHONY: all
 all: format build-dev lint test
@@ -145,4 +135,4 @@ clean:
 	rm -f .coverage.*
 	rm -rf build
 	rm -rf perf.data*
-	rm -rf pydantic_core/*.so
+	rm -rf python/pydantic_core/*.so

generate_self_schema.py

@@ -2,7 +2,7 @@
 This script generates the schema for the schema - e.g.
 a definition of what inputs can be provided to `SchemaValidator()`.
 
-The schema is generated from `pydantic_core/core_schema.py`.
+The schema is generated from `python/pydantic_core/core_schema.py`.
 """
 from __future__ import annotations as _annotations
 
@@ -34,7 +34,7 @@
 else:
     # can't import core_schema.py directly as pydantic-core might not be installed
     core_schema_spec = importlib.util.spec_from_file_location(
-        '_typing', str(THIS_DIR / 'pydantic_core' / 'core_schema.py')
+        '_typing', str(THIS_DIR / 'python' / 'pydantic_core' / 'core_schema.py')
     )
     core_schema = importlib.util.module_from_spec(core_schema_spec)
     core_schema_spec.loader.exec_module(core_schema)

pyproject.toml

@@ -44,16 +44,17 @@ Funding = 'https://github.com/sponsors/samuelcolvin'
 Source = 'https://github.com/pydantic/pydantic-core'
 
 [tool.maturin]
+python-source = "python"
 module-name = "pydantic_core._pydantic_core"
 bindings = 'pyo3'
-features = ["pyo3/extension-module", "mimalloc-allocator"]
+features = ["pyo3/extension-module"]
 
 [tool.ruff]
 line-length = 120
 extend-select = ['Q', 'RUF100', 'C90', 'I']
 flake8-quotes = {inline-quotes = 'single', multiline-quotes = 'double'}
 mccabe = { max-complexity = 13 }
-isort = { known-first-party = ['pydantic-core', 'tests'] }
+isort = { known-first-party = ['pydantic_core', 'tests'] }
 
 [tool.pytest.ini_options]
 testpaths = 'tests'

pydantic_core/__init__.py renamed to python/pydantic_core/__init__.py

@@ -4,7 +4,7 @@ extern crate core;
 
 use pyo3::prelude::*;
 
-#[cfg(feature = "mimalloc-allocator")]
+#[cfg(feature = "mimalloc")]
 #[global_allocator]
 static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
@@ -63,5 +63,9 @@ fn _pydantic_core(py: Python, m: &PyModule) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(to_json, m)?)?;
     m.add_function(wrap_pyfunction!(to_jsonable_python, m)?)?;
     m.add_function(wrap_pyfunction!(list_all_errors, m)?)?;
+
+    #[cfg(not(feature = "mimalloc"))]
+    m.setattr("__pydantic_core_default_allocator__", true)?; // uses setattr so this is not in __all__
+
     Ok(())
 }

pydantic_core/_pydantic_core.pyi renamed to python/pydantic_core/_pydantic_core.pyi

@@ -13,7 +13,7 @@ def find_examples(*_directories):
 
 
 @pytest.mark.skipif(sys.platform not in {'linux', 'darwin'}, reason='Only on linux and macos')
-@pytest.mark.parametrize('example', find_examples('pydantic_core/core_schema.py'), ids=str)
+@pytest.mark.parametrize('example', find_examples('python/pydantic_core/core_schema.py'), ids=str)
 def test_docstrings(example: CodeExample, eval_example: EvalExample):
     eval_example.set_config(quotes='single')
 

pydantic_core/core_schema.py renamed to python/pydantic_core/core_schema.py

@@ -402,7 +402,7 @@ def test_all_errors():
     error_types = [e['type'] for e in errors]
     if error_types != list(core_schema.ErrorType.__args__):
         literal = ''.join(f'\n    {e!r},' for e in error_types)
-        print(f'python code (end of pydantic_core/core_schema.py):\n\nErrorType = Literal[{literal}\n]')
+        print(f'python code (end of python/pydantic_core/core_schema.py):\n\nErrorType = Literal[{literal}\n]')
         pytest.fail('core_schema.ErrorType needs to be updated')
 
 

pydantic_core/py.typed renamed to python/pydantic_core/py.typed

@@ -160,7 +160,9 @@ def get_type_value(schema):
     schema_types = tuple(dict.fromkeys(schema_types))  # remove duplicates while preserving order
     if get_args(CoreSchemaType) != schema_types:
         literal = ''.join(f'\n    {e!r},' for e in schema_types)
-        print(f'python code (near end of pydantic_core/core_schema.py):\n\nCoreSchemaType = Literal[{literal}\n]')
+        print(
+            f'python code (near end of python/pydantic_core/core_schema.py):\n\nCoreSchemaType = Literal[{literal}\n]'
+        )
         pytest.fail('core_schema.CoreSchemaType needs to be updated')