import logging
import time
from django_pyforge import ModelSchema, serialize_instance
logger = logging.getLogger("django_pyforge")
_n1_warned = set()
_UNINITIALIZED = object()
_field_cache = {}
class _PyForgeListSerializer:
def __init__(self, child_class, instances, rust_fields, python_field_names,
schema, ordered_field_names):
self.child_class = child_class
self.instances = instances
self.rust_fields = rust_fields
self.python_field_names = python_field_names
self.schema = schema
self.ordered_field_names = ordered_field_names
@property
def data(self):
t0 = time.perf_counter()
results = []
schema = self.schema
rust_fields = self.rust_fields
python_field_names = self.python_field_names
ordered = self.ordered_field_names
has_python = bool(python_field_names)
if has_python and self.instances:
_detect_n1(self.instances[0], python_field_names)
for instance in self.instances:
try:
row = serialize_instance(instance, schema)
except Exception:
child = self.child_class(instance)
results.append(child.data)
continue
if has_python:
for field_name in python_field_names:
if field_name == "id":
row["id"] = instance.pk
elif hasattr(instance, field_name + "_id"):
row[field_name] = getattr(instance, field_name + "_id")
else:
row[field_name] = getattr(instance, field_name, None)
if ordered:
row = {k: row[k] for k in ordered if k in row}
results.append(row)
elapsed_ms = (time.perf_counter() - t0) * 1000
try:
from django_pyforge.middleware import record_serialization
record_serialization(
len(rust_fields) * len(results),
len(python_field_names) * len(results),
elapsed_ms,
)
except Exception:
pass
return results
class RustSerializerMixin:
_pyforge_schema = _UNINITIALIZED
@classmethod
def _init_pyforge_schema(cls):
if cls._pyforge_schema is not _UNINITIALIZED:
return
try:
model_class = cls.Meta.model
except AttributeError:
cls._pyforge_schema = None
return
try:
cls._pyforge_schema = ModelSchema(model_class)
except Exception as exc:
logger.debug("PyForge: could not compile schema for %s: %s",
model_class.__name__, exc)
cls._pyforge_schema = None
@classmethod
def _get_field_classification(cls, serializer_instance):
cache_key = id(cls)
cached = _field_cache.get(cache_key)
if cached is not None:
return cached
cls._init_pyforge_schema()
if cls._pyforge_schema is None:
result = (set(), set(serializer_instance.fields.keys()),
list(serializer_instance.fields.keys()))
_field_cache[cache_key] = result
return result
rust_schema_fields = set(cls._pyforge_schema.field_names_list)
rust_fields = set()
python_fields = set()
for field_name, field_obj in serializer_instance.fields.items():
field_class_name = type(field_obj).__name__
is_computed = field_class_name in (
"SerializerMethodField",
"HiddenField",
"ReadOnlyField",
)
is_nested = hasattr(field_obj, "Meta") and hasattr(field_obj, "fields")
source = getattr(field_obj, "source", field_name)
has_custom_source = source != field_name and source != "*"
in_rust_schema = field_name in rust_schema_fields
if in_rust_schema and not is_computed and not is_nested and not has_custom_source:
rust_fields.add(field_name)
else:
python_fields.add(field_name)
ordered = list(serializer_instance.fields.keys())
result = (rust_fields, python_fields, ordered)
_field_cache[cache_key] = result
return result
def __init_subclass__(cls, **kwargs):
super().__init_subclass__(**kwargs)
_field_cache.pop(id(cls), None)
def __class_getitem__(cls, params):
return cls
@classmethod
def many_init(cls, *args, **kwargs):
instances = args[0] if args else kwargs.get("instance")
many = kwargs.pop("many", False)
if not many or instances is None:
return super(RustSerializerMixin, cls).many_init(*args, **kwargs)
cls._init_pyforge_schema()
if cls._pyforge_schema is None:
kwargs["many"] = True
return super(RustSerializerMixin, cls).many_init(*args, **kwargs)
try:
temp = cls()
rust_fields, python_fields, ordered = cls._get_field_classification(temp)
except Exception:
kwargs["many"] = True
return super(RustSerializerMixin, cls).many_init(*args, **kwargs)
total = len(rust_fields) + len(python_fields)
rust_ratio = len(rust_fields) / total if total > 0 else 0
if rust_ratio >= 0.80:
try:
return _PyForgeListSerializer(
child_class=cls,
instances=instances,
rust_fields=rust_fields,
python_field_names=python_fields,
schema=cls._pyforge_schema,
ordered_field_names=ordered,
)
except Exception:
pass
kwargs["many"] = True
return super(RustSerializerMixin, cls).many_init(*args, **kwargs)
def to_representation(self, instance):
cls = type(self)
cls._init_pyforge_schema()
if cls._pyforge_schema is None:
return super().to_representation(instance)
rust_fields, python_fields, ordered = cls._get_field_classification(self)
if not rust_fields:
return super().to_representation(instance)
try:
rust_result = serialize_instance(instance, cls._pyforge_schema)
if not python_fields:
return {k: rust_result[k] for k in ordered if k in rust_result}
result = {}
for field_name in rust_fields:
if field_name in rust_result:
result[field_name] = rust_result[field_name]
for field_name in python_fields:
if field_name == "id":
result[field_name] = instance.pk
elif hasattr(instance, field_name + "_id"):
result[field_name] = getattr(instance, field_name + "_id")
else:
field_obj = self.fields[field_name]
try:
attr = field_obj.get_attribute(instance)
result[field_name] = field_obj.to_representation(attr)
except Exception:
result[field_name] = None
return {k: result[k] for k in ordered if k in result}
except Exception as exc:
logger.debug("PyForge: Rust serialization failed, falling back to DRF: %s", exc)
return super().to_representation(instance)
def _detect_n1(instance, python_field_names):
try:
from django.conf import settings
if not settings.DEBUG:
return
except Exception:
return
for field_name in python_field_names:
fk_attr = field_name + "_id"
if not hasattr(instance, fk_attr):
continue
cache = getattr(instance, "_state", None)
if cache is None:
continue
fields_cache = getattr(cache, "fields_cache", {})
if field_name not in fields_cache:
cache_key = f"{type(instance).__name__}.{field_name}"
if cache_key not in _n1_warned:
_n1_warned.add(cache_key)
logger.warning(
"PyForge N+1 detected: field '%s' accesses ForeignKey '%s' "
"which is not in select_related. Add "
".select_related('%s') to your queryset.",
field_name, field_name, field_name,
)
def serialize_stream(queryset, schema, chunk_size=1000):
from django_pyforge import serialize_instance
iterator = queryset.iterator(chunk_size=chunk_size)
chunk = []
for instance in iterator:
try:
chunk.append(serialize_instance(instance, schema))
except Exception:
continue
if len(chunk) >= chunk_size:
yield chunk
chunk = []
if chunk:
yield chunk