Initial stubtest testing with autogenerated allowlist

.github/workflows/test.yml

@@ -9,6 +9,7 @@ on:
 
 jobs:
   mypy-self-check:
+    timeout-minutes: 10
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -31,6 +32,7 @@ jobs:
         run: mypy --cache-dir=/dev/null --no-incremental rest_framework-stubs
 
   test:
+    timeout-minutes: 10
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -52,3 +54,28 @@ jobs:
 
       - name: Run tests
         run: pytest
+
+  stubtest:
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.12']
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install binutils libproj-dev gdal-bin
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          pip install -U pip setuptools wheel
+          SETUPTOOLS_ENABLE_FEATURES=legacy-editable pip install -r ./requirements.txt
+
+      - name: Run stubtest
+        run: bash ./scripts/stubtest.sh

CONTRIBUTING.md

@@ -7,6 +7,7 @@ This project is open source and community driven. As such we encourage code cont
 3. Improve plugin code and extend its capabilities
 4. Write tests
 5. Update dependencies
+6. Fix and remove things from our `scripts/stubtest/allowlist_todo.txt`
 
 Type stubs in `.pyi` files should follow
 [coding conventions from typeshed project](https://github.com/python/typeshed/blob/main/CONTRIBUTING.md#conventions).
@@ -73,6 +74,19 @@ To execute the unit tests, simply run:
 pytest
 ```
 
+### Testing stubs with `stubtest`
+
+Run `bash ./scripts/stubtest.sh` to test that stubs and sources are in-line.
+
+We have two special files to allow errors:
+1. `scripts/stubtest/allowlist.txt` where we store things that we really don't care about: hacks, DRF internal utility modules, things that are handled by our plugin, things that are not representable by type system, etc
+2. `scripts/stubtest/allowlist_todo.txt` where we store all errors there are right now. Basically, this is a TODO list: we need to work through this list and fix things (or move entries to real `allowlist.txt`). In the end, ideally we can remove this file
+
+You might also want to disable `incremental` mode while working on `stubtest` changes.
+This mode leads to several known problems (stubs do not show up or have strange errors).
+
+**Important**: right now we only run `stubtest` on Python 3.12 (because it is the latest released version at the moment), any other versions might generate different outputs. Any work to create per-version allowlists is welcome.
+
 ## Submission Guidelines
 
 The workflow for contributions is fairly simple:

scripts/stubtest.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# Run this script as `bash ./scripts/stubtest.sh`
+
+set -e
+
+export MYPYPATH='.'
+
+# TODO: remove `--ignore-positional-only` when ready
+stubtest rest_framework \
+    --mypy-config-file mypy.ini \
+    --ignore-positional-only \
+    --allowlist scripts/stubtest/allowlist.txt \
+    --allowlist scripts/stubtest/allowlist_todo.txt

scripts/stubtest/allowlist.txt

@@ -0,0 +1,6 @@
+# This is a true allow list with things that we really don't care about.
+# `allowlist_todo.txt` is autogenerated by `stubtest --generate-allowlist`
+# and might contain actual problems and things that we *do want* to fix.
+#
+# Please, move things here when you are sure that they really should be ignored.
+# Comments about why things are ignored are mandatory.

scripts/stubtest/allowlist_todo.txt

@@ -0,0 +1,207 @@
+# Autogenerated by `stubtest`
+# Unsorted: there are real problems and things we can really ignore.
+
+rest_framework.RemovedInDRF313Warning
+rest_framework.RemovedInDRF314Warning
+rest_framework.RemovedInDRF315Warning
+rest_framework.__license__
+rest_framework.__title__
+rest_framework.authtoken.admin.TokenAdmin
+rest_framework.authtoken.admin.TokenChangeList
+rest_framework.authtoken.default_app_config
+rest_framework.authtoken.management.commands.drf_create_token.UserModel
+rest_framework.authtoken.migrations
+rest_framework.authtoken.migrations.0001_initial
+rest_framework.authtoken.migrations.0002_auto_20160226_1747
+rest_framework.authtoken.migrations.0003_tokenproxy
+rest_framework.authtoken.models.Token.created
+rest_framework.authtoken.models.Token.get_next_by_created
+rest_framework.authtoken.models.Token.get_previous_by_created
+rest_framework.authtoken.models.Token.key
+rest_framework.authtoken.models.Token.user
+rest_framework.authtoken.models.Token.user_id
+rest_framework.authtoken.models.TokenProxy.get_next_by_created
+rest_framework.authtoken.models.TokenProxy.get_previous_by_created
+rest_framework.authtoken.models.TokenProxy.pk
+rest_framework.authtoken.models.TokenProxy.user_id
+rest_framework.authtoken.views.ObtainAuthToken.get_serializer
+rest_framework.authtoken.views.ObtainAuthToken.get_serializer_context
+rest_framework.authtoken.views.ObtainAuthToken.serializer_class
+rest_framework.authtoken.views.obtain_auth_token
+rest_framework.compat.CodeBlockPreprocessor
+rest_framework.compat.Preprocessor
+rest_framework.compat.QuerySet
+rest_framework.compat.__all__
+rest_framework.compat.apply_markdown
+rest_framework.decorators.ViewSetAction
+rest_framework.decorators.action
+rest_framework.default_app_config
+rest_framework.documentation.get_docs_view
+rest_framework.documentation.get_schemajs_view
+rest_framework.documentation.include_docs_urls
+rest_framework.fields.BooleanField.initial
+rest_framework.fields.CharField.initial
+rest_framework.fields.DateField.__init__
+rest_framework.fields.DateField.to_internal_value
+rest_framework.fields.DateTimeField.__init__
+rest_framework.fields.DateTimeField.to_internal_value
+rest_framework.fields.DecimalField.__init__
+rest_framework.fields.DictField.__init__
+rest_framework.fields.DictField.initial
+rest_framework.fields.DurationField.to_internal_value
+rest_framework.fields.Field.__init__
+rest_framework.fields.FilePathField.__init__
+rest_framework.fields.HStoreField.__init__
+rest_framework.fields.HiddenField.__init__
+rest_framework.fields.JSONField.__init__
+rest_framework.fields.ListField.__init__
+rest_framework.fields.ListField.initial
+rest_framework.fields.ListField.to_representation
+rest_framework.fields.ModelField.get_attribute
+rest_framework.fields.ModelField.to_representation
+rest_framework.fields.MultipleChoiceField.__init__
+rest_framework.fields.NullBooleanField
+rest_framework.fields.Option
+rest_framework.fields.REGEX_TYPE
+rest_framework.fields.ReadOnlyField.__init__
+rest_framework.fields.SupportsToPython
+rest_framework.fields.TimeField.__init__
+rest_framework.fields.TimeField.to_internal_value
+rest_framework.fields._UnvalidatedField.__init__
+rest_framework.fields.empty
+rest_framework.generics.BaseFilterProtocol
+rest_framework.generics.UsesQuerySet
+rest_framework.negotiation.DefaultContentNegotiation.settings
+rest_framework.pagination.HtmlContext
+rest_framework.pagination.HtmlContextWithPageLinks
+rest_framework.parsers.BaseParser.media_type
+rest_framework.parsers.FileUploadParser.get_encoded_filename
+rest_framework.relations.HyperlinkedIdentityField.__init__
+rest_framework.relations.HyperlinkedRelatedField.__init__
+rest_framework.relations.HyperlinkedRelatedField.get_object
+rest_framework.relations.ManyRelatedField.__init__
+rest_framework.relations.ManyRelatedField.initial
+rest_framework.relations.ManyRelatedField.to_representation
+rest_framework.relations.PrimaryKeyRelatedField.__init__
+rest_framework.relations.RelatedField.__init__
+rest_framework.relations.SlugRelatedField.__init__
+rest_framework.relations.SlugRelatedField.to_representation
+rest_framework.relations.StringRelatedField.__init__
+rest_framework.renderers.BaseRenderer.format
+rest_framework.renderers.BaseRenderer.media_type
+rest_framework.renderers.BrowsableAPIRenderer.get_extra_actions
+rest_framework.renderers.CoreAPIJSONOpenAPIRenderer.ensure_ascii
+rest_framework.renderers.CoreJSONRenderer.render
+rest_framework.renderers.JSONOpenAPIRenderer.encoder_class
+rest_framework.renderers.JSONOpenAPIRenderer.ensure_ascii
+rest_framework.renderers._BaseOpenAPIRenderer.__init__
+rest_framework.renderers._BaseOpenAPIRenderer.render
+rest_framework.request.Request.DATA
+rest_framework.request.Request.QUERY_PARAMS
+rest_framework.routers.BaseRouter
+rest_framework.routers.BaseRouter.register
+rest_framework.routers.DefaultRouter
+rest_framework.routers.DefaultRouter.APIRootView
+rest_framework.routers.DefaultRouter.APISchemaView
+rest_framework.routers.DefaultRouter.SchemaGenerator
+rest_framework.routers.RenameRouterMethods
+rest_framework.routers.SimpleRouter
+rest_framework.schemas.SchemaGenerator.__init__
+rest_framework.schemas.coreapi.SchemaGenerator.__init__
+rest_framework.schemas.generators.common_path
+rest_framework.schemas.generators.endpoint_ordering
+rest_framework.schemas.get_schema_view
+rest_framework.schemas.openapi.AutoSchema.__init__
+rest_framework.schemas.openapi.AutoSchema.get_reference
+rest_framework.schemas.openapi.AutoSchema.get_request_serializer
+rest_framework.schemas.openapi.AutoSchema.get_response_serializer
+rest_framework.schemas.openapi.DRFOpenAPIInfo
+rest_framework.schemas.openapi.DRFOpenAPISchema
+rest_framework.schemas.openapi.ExternalDocumentationObject
+rest_framework.schemas.openapi.SchemaGenerator.get_schema
+rest_framework.schemas.views.SchemaView.__init__
+rest_framework.schemas.views.SchemaView.get
+rest_framework.schemas.views.SchemaView.renderer_classes
+rest_framework.serializers.APIException
+rest_framework.serializers.AuthenticationFailed
+rest_framework.serializers.BaseSerializer.__init__
+rest_framework.serializers.BaseSerializer.is_valid
+rest_framework.serializers.BooleanField.initial
+rest_framework.serializers.CharField.initial
+rest_framework.serializers.DateField.__init__
+rest_framework.serializers.DateField.to_internal_value
+rest_framework.serializers.DateTimeField.__init__
+rest_framework.serializers.DateTimeField.to_internal_value
+rest_framework.serializers.DecimalField.__init__
+rest_framework.serializers.DictField.__init__
+rest_framework.serializers.DictField.initial
+rest_framework.serializers.DurationField.to_internal_value
+rest_framework.serializers.Field.__init__
+rest_framework.serializers.FilePathField.__init__
+rest_framework.serializers.HStoreField.__init__
+rest_framework.serializers.HiddenField.__init__
+rest_framework.serializers.HyperlinkedIdentityField.__init__
+rest_framework.serializers.HyperlinkedRelatedField.__init__
+rest_framework.serializers.HyperlinkedRelatedField.get_object
+rest_framework.serializers.JSONField.__init__
+rest_framework.serializers.ListField.__init__
+rest_framework.serializers.ListField.initial
+rest_framework.serializers.ListField.to_representation
+rest_framework.serializers.ListSerializer.is_valid
+rest_framework.serializers.ListSerializer.to_representation
+rest_framework.serializers.ManyRelatedField.__init__
+rest_framework.serializers.ManyRelatedField.initial
+rest_framework.serializers.ManyRelatedField.to_representation
+rest_framework.serializers.MethodNotAllowed
+rest_framework.serializers.ModelField.get_attribute
+rest_framework.serializers.ModelField.to_representation
+rest_framework.serializers.ModelSerializer.Meta
+rest_framework.serializers.ModelSerializer.__init__
+rest_framework.serializers.MultipleChoiceField.__init__
+rest_framework.serializers.NotAcceptable
+rest_framework.serializers.NotAuthenticated
+rest_framework.serializers.NotFound
+rest_framework.serializers.NullBooleanField
+rest_framework.serializers.ParseError
+rest_framework.serializers.PermissionDenied
+rest_framework.serializers.PrimaryKeyRelatedField.__init__
+rest_framework.serializers.ReadOnlyField.__init__
+rest_framework.serializers.RelatedField.__init__
+rest_framework.serializers.SlugRelatedField.__init__
+rest_framework.serializers.SlugRelatedField.to_representation
+rest_framework.serializers.StringRelatedField.__init__
+rest_framework.serializers.Throttled
+rest_framework.serializers.TimeField.__init__
+rest_framework.serializers.TimeField.to_internal_value
+rest_framework.serializers.UnsupportedMediaType
+rest_framework.serializers.empty
+rest_framework.settings.DefaultsSettings
+rest_framework.settings.api_settings
+rest_framework.status.HTTP_102_PROCESSING
+rest_framework.status.HTTP_103_EARLY_HINTS
+rest_framework.status.HTTP_421_MISDIRECTED_REQUEST
+rest_framework.status.HTTP_425_TOO_EARLY
+rest_framework.templatetags.rest_framework.urlize_quoted_links
+rest_framework.test.APIClient.options
+rest_framework.test.APIRequestFactory.delete
+rest_framework.test.APIRequestFactory.get
+rest_framework.test.APIRequestFactory.options
+rest_framework.test.APIRequestFactory.patch
+rest_framework.test.APIRequestFactory.post
+rest_framework.test.APIRequestFactory.put
+rest_framework.test.RequestsClient.__init__
+rest_framework.test.cleanup_url_patterns
+rest_framework.throttling.SimpleRateThrottle.cache
+rest_framework.utils.encoders.JSONEncoder.default
+rest_framework.utils.field_mapping.get_unique_error_message
+rest_framework.utils.formatting.lazy_format.__mod__
+rest_framework.utils.json.dump
+rest_framework.utils.json.dumps
+rest_framework.utils.json.load
+rest_framework.utils.json.loads
+rest_framework.validators.BaseUniqueForValidator.message
+rest_framework.validators.ContextValidator
+rest_framework.validators.Validator
+rest_framework.views.APIView.metadata_class
+rest_framework.views.AsView
+rest_framework.views.GenericView