diff --git a/django/db/models/query.py b/django/db/models/query.py
index dca6eea1ba..ac68bec0dd 100644
--- a/django/db/models/query.py
+++ b/django/db/models/query.py
@@ -1906,17 +1906,20 @@ def prefetch_one_level(instances, prefetcher, lookup, level):
         rel_attr_val = rel_obj_attr(rel_obj)
         rel_obj_cache.setdefault(rel_attr_val, []).append(rel_obj)
 
+    to_attr, as_attr = lookup.get_current_to_attr(level)
+    # Make sure `to_attr` does not conflict with a field.
+    if as_attr and instances:
+        # We assume that objects retrieved are homogeneous (which is the premise
+        # of prefetch_related), so what applies to first object applies to all.
+        model = instances[0].__class__
+        for related_m2m in model._meta.get_all_related_many_to_many_objects():
+            if related_m2m.get_accessor_name() == to_attr:
+                msg = 'to_attr={} conflicts with a field on the {} model.'
+                raise ValueError(msg.format(to_attr, model.__name__))
+
     for obj in instances:
         instance_attr_val = instance_attr(obj)
         vals = rel_obj_cache.get(instance_attr_val, [])
-        to_attr, as_attr = lookup.get_current_to_attr(level)
-
-        # Check we are not shadowing a field on obj.
-        if as_attr:
-            for related_m2m in obj._meta.get_all_related_many_to_many_objects():
-                if related_m2m.get_accessor_name() == to_attr:
-                    msg = 'to_attr={} conflicts with a field on the {} model.'
-                    raise ValueError(msg.format(to_attr, obj.__class__.__name__))
 
         if single:
             val = vals[0] if vals else None