Merge remote-tracking branch 'origin/3.3.x' into 4.0.x

.doctrine-project.json

@@ -35,17 +35,23 @@
             "slug": "3.0",
             "maintained": false
         },
+        {
+            "name": "2.21",
+            "branchName": "2.21.x",
+            "slug": "2.21",
+            "upcoming": true
+        },
         {
             "name": "2.20",
             "branchName": "2.20.x",
             "slug": "2.20",
-            "upcoming": true
+            "maintained": true
         },
         {
             "name": "2.19",
             "branchName": "2.19.x",
             "slug": "2.19",
-            "maintained": true
+            "maintained": false
         },
         {
             "name": "2.18",

UPGRADE.md

@@ -783,6 +783,15 @@ Use `toIterable()` instead.
 
 # Upgrade to 2.20
 
+## Add `Doctrine\ORM\Query\OutputWalker` interface, deprecate `Doctrine\ORM\Query\SqlWalker::getExecutor()`
+
+Output walkers should implement the new `\Doctrine\ORM\Query\OutputWalker` interface and create
+`Doctrine\ORM\Query\Exec\SqlFinalizer` instances instead of `Doctrine\ORM\Query\Exec\AbstractSqlExecutor`s.
+The output walker must not base its workings on the query `firstResult`/`maxResult` values, so that the 
+`SqlFinalizer` can be kept in the query cache and used regardless of the actual `firstResult`/`maxResult` values.
+Any operation dependent on `firstResult`/`maxResult` should take place within the `SqlFinalizer::createExecutor()`
+method. Details can be found at https://github.com/doctrine/orm/pull/11188.
+
 ## Explictly forbid property hooks
 
 Property hooks are not supported yet by Doctrine ORM. Until support is added,
@@ -791,10 +800,16 @@ change in behavior.
 
 Progress on this is tracked at https://github.com/doctrine/orm/issues/11624 .
 
-## PARTIAL DQL syntax is undeprecated for non-object hydration
+## PARTIAL DQL syntax is undeprecated 
+
+Use of the PARTIAL keyword is not deprecated anymore in DQL, because we will be
+able to support PARTIAL objects with PHP 8.4 Lazy Objects and
+Symfony/VarExporter in a better way. When we decided to remove this feature
+these two abstractions did not exist yet.
 
-Use of the PARTIAL keyword is not deprecated anymore in DQL when used with a hydrator
-that is not creating entities, such as the ArrayHydrator.
+WARNING: If you want to upgrade to 3.x and still use PARTIAL keyword in DQL
+with array or object hydrators, then you have to directly migrate to ORM 3.3.x or higher.
+PARTIAL keyword in DQL is not available in 3.0, 3.1 and 3.2 of ORM.
 
 ## Deprecate `\Doctrine\ORM\Query\Parser::setCustomOutputTreeWalker()`
 

docs/en/index.rst

@@ -74,6 +74,7 @@ Advanced Topics
 * :doc:`Improving Performance <reference/improving-performance>`
 * :doc:`Caching <reference/caching>`
 * :doc:`Partial Hydration <reference/partial-hydration>`
+* :doc:`Partial Objects <reference/partial-objects>`
 * :doc:`Change Tracking Policies <reference/change-tracking-policies>`
 * :doc:`Best Practices <reference/best-practices>`
 * :doc:`Metadata Drivers <reference/metadata-drivers>`

docs/en/reference/dql-doctrine-query-language.rst

@@ -533,14 +533,23 @@ back. Instead, you receive only arrays as a flat rectangular result
 set, similar to how you would if you were just using SQL directly
 and joining some data.
 
-If you want to select a partial number of fields for hydration entity in
-the context of array hydration and joins you can use the ``partial`` DQL keyword:
+If you want to select partial objects or fields in array hydration you can use the ``partial``
+DQL keyword:
+
+.. code-block:: php
+
+    <?php
+    $query = $em->createQuery('SELECT partial u.{id, username} FROM CmsUser u');
+    $users = $query->getResult(); // array of partially loaded CmsUser objects
+
+You can use the partial syntax when joining as well:
 
 .. code-block:: php
 
     <?php
     $query = $em->createQuery('SELECT partial u.{id, username}, partial a.{id, name} FROM CmsUser u JOIN u.articles a');
-    $users = $query->getArrayResult(); // array of partially loaded CmsUser and CmsArticle fields
+    $usersArray = $query->getArrayResult(); // array of partially loaded CmsUser and CmsArticle fields
+    $users = $query->getResult(); // array of partially loaded CmsUser objects
 
 "NEW" Operator Syntax
 ^^^^^^^^^^^^^^^^^^^^^
@@ -591,23 +600,80 @@ You can also nest several DTO :
             // Bind values to the object properties.
         }
     }
-	
+
     class AddressDTO
     {
         public function __construct(string $street, string $city, string $zip)
         {
             // Bind values to the object properties.
         }
     }
-	
+
 .. code-block:: php
 
     <?php
     $query = $em->createQuery('SELECT NEW CustomerDTO(c.name, e.email, NEW AddressDTO(a.street, a.city, a.zip)) FROM Customer c JOIN c.email e JOIN c.address a');
     $users = $query->getResult(); // array of CustomerDTO
-	
+
 Note that you can only pass scalar expressions or other Data Transfer Objects to the constructor.
 
+If you use your data transfer objects for multiple queries, and you would rather not have to
+specify arguments that precede the ones you are really interested in, you can use named arguments.
+
+Consider the following DTO, which uses optional arguments:
+
+.. code-block:: php
+
+    <?php
+
+    class CustomerDTO
+    {
+        public function __construct(
+            public string|null $name = null,
+            public string|null $email = null,
+            public string|null $city = null,
+            public mixed|null $value = null,
+            public AddressDTO|null $address = null,
+        ) {
+        }
+    }
+
+You can specify arbitrary arguments in an arbitrary order by using the named argument syntax, and the ORM will try to match argument names with the selected column names.
+The syntax relies on the NAMED keyword, like so:
+
+.. code-block:: php
+
+    <?php
+    $query = $em->createQuery('SELECT NEW NAMED CustomerDTO(a.city, c.name) FROM Customer c JOIN c.address a');
+    $users = $query->getResult(); // array of CustomerDTO
+
+    // CustomerDTO => {name : 'SMITH', email: null, city: 'London', value: null}
+
+ORM will also give precedence to column aliases over column names :
+
+.. code-block:: php
+
+    <?php
+    $query = $em->createQuery('SELECT NEW NAMED CustomerDTO(c.name, CONCAT(a.city, ' ' , a.zip) AS value) FROM Customer c JOIN c.address a');
+    $users = $query->getResult(); // array of CustomerDTO
+
+    // CustomerDTO => {name : 'DOE', email: null, city: null, value: 'New York 10011'}
+
+To define a custom name for a DTO constructor argument, you can either alias the column with the ``AS`` keyword.
+
+The ``NAMED`` keyword must precede all DTO you want to instantiate :
+
+.. code-block:: php
+
+    <?php
+    $query = $em->createQuery('SELECT NEW NAMED CustomerDTO(c.name, NEW NAMED AddressDTO(a.street, a.city, a.zip) AS address) FROM Customer c JOIN c.address a');
+    $users = $query->getResult(); // array of CustomerDTO
+
+    // CustomerDTO => {name : 'DOE', email: null, city: null, value: 'New York 10011'}
+
+If two arguments have the same name, a ``DuplicateFieldException`` is thrown.
+If a field cannot be matched with a property name, a ``NoMatchingPropertyException`` is thrown. This typically happens when using functions without aliasing them.
+
 Using INDEX BY
 ~~~~~~~~~~~~~~
 
@@ -1370,6 +1436,15 @@ exist mostly internal query hints that are not be consumed in
 userland. However the following few hints are to be used in
 userland:
 
+
+-  ``Query::HINT_FORCE_PARTIAL_LOAD`` - Allows to hydrate objects
+   although not all their columns are fetched. This query hint can be
+   used to handle memory consumption problems with large result-sets
+   that contain char or binary data. Doctrine has no way of implicitly
+   reloading this data. Partially loaded objects have to be passed to
+   ``EntityManager::refresh()`` if they are to be reloaded fully from
+   the database. This query hint is deprecated and will be removed
+   in the future (\ `Details <https://github.com/doctrine/orm/issues/8471>`_)
 -  ``Query::HINT_REFRESH`` - This query is used internally by
    ``EntityManager::refresh()`` and can be used in userland as well.
    If you specify this hint and a query returns the data for an entity
@@ -1627,7 +1702,7 @@ Select Expressions
     PartialObjectExpression ::= "PARTIAL" IdentificationVariable "." PartialFieldSet
     PartialFieldSet         ::= "{" SimpleStateField {"," SimpleStateField}* "}"
     NewObjectExpression     ::= "NEW" AbstractSchemaName "(" NewObjectArg {"," NewObjectArg}* ")"
-    NewObjectArg            ::= ScalarExpression | "(" Subselect ")" | NewObjectExpression
+    NewObjectArg            ::= (ScalarExpression | "(" Subselect ")" | NewObjectExpression) ["AS" AliasResultVariable]
 
 Conditional Expressions
 ~~~~~~~~~~~~~~~~~~~~~~~

docs/en/reference/partial-hydration.rst

@@ -1,11 +1,6 @@
 Partial Hydration
 =================
 
-.. note::
-
-    Creating Partial Objects through DQL was possible in ORM 2,
-    but is only supported for array hydration as of ORM 3.
-
 Partial hydration of entities is allowed in the array hydrator, when
 only a subset of the fields of an entity are loaded from the database
 and the nested results are still created based on the entity relationship structure.

docs/en/reference/partial-objects.rst

@@ -0,0 +1,88 @@
+Partial Objects
+===============
+
+A partial object is an object whose state is not fully initialized
+after being reconstituted from the database and that is
+disconnected from the rest of its data. The following section will
+describe why partial objects are problematic and what the approach
+of Doctrine to this problem is.
+
+.. note::
+
+    The partial object problem in general does not apply to
+    methods or queries where you do not retrieve the query result as
+    objects. Examples are: ``Query#getArrayResult()``,
+    ``Query#getScalarResult()``, ``Query#getSingleScalarResult()``,
+    etc.
+
+.. warning::
+
+    Use of partial objects is tricky. Fields that are not retrieved
+    from the database will not be updated by the UnitOfWork even if they
+    get changed in your objects. You can only promote a partial object
+    to a fully-loaded object by calling ``EntityManager#refresh()``
+    or a DQL query with the refresh flag.
+
+
+What is the problem?
+--------------------
+
+In short, partial objects are problematic because they are usually
+objects with broken invariants. As such, code that uses these
+partial objects tends to be very fragile and either needs to "know"
+which fields or methods can be safely accessed or add checks around
+every field access or method invocation. The same holds true for
+the internals, i.e. the method implementations, of such objects.
+You usually simply assume the state you need in the method is
+available, after all you properly constructed this object before
+you pushed it into the database, right? These blind assumptions can
+quickly lead to null reference errors when working with such
+partial objects.
+
+It gets worse with the scenario of an optional association (0..1 to
+1). When the associated field is NULL, you don't know whether this
+object does not have an associated object or whether it was simply
+not loaded when the owning object was loaded from the database.
+
+These are reasons why many ORMs do not allow partial objects at all
+and instead you always have to load an object with all its fields
+(associations being proxied). One secure way to allow partial
+objects is if the programming language/platform allows the ORM tool
+to hook deeply into the object and instrument it in such a way that
+individual fields (not only associations) can be loaded lazily on
+first access. This is possible in Java, for example, through
+bytecode instrumentation. In PHP though this is not possible, so
+there is no way to have "secure" partial objects in an ORM with
+transparent persistence.
+
+Doctrine, by default, does not allow partial objects. That means,
+any query that only selects partial object data and wants to
+retrieve the result as objects (i.e. ``Query#getResult()``) will
+raise an exception telling you that partial objects are dangerous.
+If you want to force a query to return you partial objects,
+possibly as a performance tweak, you can use the ``partial``
+keyword as follows:
+
+.. code-block:: php
+
+    <?php
+    $q = $em->createQuery("select partial u.{id,name} from MyApp\Domain\User u");
+
+You can also get a partial reference instead of a proxy reference by
+calling:
+
+.. code-block:: php
+
+    <?php
+    $reference = $em->getPartialReference('MyApp\Domain\User', 1);
+
+Partial references are objects with only the identifiers set as they
+are passed to the second argument of the ``getPartialReference()`` method.
+All other fields are null.
+
+When should I force partial objects?
+------------------------------------
+
+Mainly for optimization purposes, but be careful of premature
+optimization as partial objects lead to potentially more fragile
+code.

docs/en/sidebar.rst

@@ -38,6 +38,7 @@
    reference/native-sql
    reference/change-tracking-policies
    reference/partial-hydration
+   reference/partial-objects
    reference/attributes-reference
    reference/xml-mapping
    reference/php-mapping

psalm-baseline.xml

@@ -207,14 +207,14 @@
     </PossiblyNullArgument>
   </file>
   <file src="src/Internal/Hydration/AbstractHydrator.php">
-    <ReferenceConstraintViolation>
-      <code><![CDATA[return $rowData;]]></code>
-      <code><![CDATA[return $rowData;]]></code>
-    </ReferenceConstraintViolation>
     <PossiblyUndefinedArrayOffset>
       <code><![CDATA[$newObject['args']]]></code>
       <code><![CDATA[$newObject['args']]]></code>
     </PossiblyUndefinedArrayOffset>
+    <ReferenceConstraintViolation>
+      <code><![CDATA[return $rowData;]]></code>
+      <code><![CDATA[return $rowData;]]></code>
+    </ReferenceConstraintViolation>
   </file>
   <file src="src/Internal/Hydration/ArrayHydrator.php">
     <PossiblyInvalidArgument>
@@ -396,6 +396,7 @@
   <file src="src/Mapping/DefaultTypedFieldMapper.php">
     <LessSpecificReturnStatement>
       <code><![CDATA[$mapping]]></code>
+      <code><![CDATA[$mapping]]></code>
     </LessSpecificReturnStatement>
     <MoreSpecificReturnType>
       <code><![CDATA[array]]></code>
@@ -885,6 +886,9 @@
     <ArgumentTypeCoercion>
       <code><![CDATA[$stringPattern]]></code>
     </ArgumentTypeCoercion>
+    <DeprecatedMethod>
+      <code><![CDATA[setSqlExecutor]]></code>
+    </DeprecatedMethod>
     <InvalidNullableReturnType>
       <code><![CDATA[AST\SelectStatement|AST\UpdateStatement|AST\DeleteStatement]]></code>
     </InvalidNullableReturnType>
@@ -1075,6 +1079,12 @@
     </RedundantConditionGivenDocblockType>
   </file>
   <file src="src/Tools/Pagination/LimitSubqueryOutputWalker.php">
+    <InvalidReturnStatement>
+      <code><![CDATA[$abstractSqlExecutor->getSqlStatements()]]></code>
+    </InvalidReturnStatement>
+    <InvalidReturnType>
+      <code><![CDATA[string]]></code>
+    </InvalidReturnType>
     <PossiblyFalseArgument>
       <code><![CDATA[strrpos($orderByItemString, ' ')]]></code>
     </PossiblyFalseArgument>

src/Cache/DefaultQueryCache.php

@@ -16,6 +16,7 @@
 use Doctrine\ORM\PersistentCollection;
 use Doctrine\ORM\Query;
 use Doctrine\ORM\Query\ResultSetMapping;
+use Doctrine\ORM\Query\SqlWalker;
 use Doctrine\ORM\UnitOfWork;
 
 use function array_map;
@@ -210,6 +211,10 @@ public function put(QueryCacheKey $key, ResultSetMapping $rsm, mixed $result, ar
             throw FeatureNotImplemented::nonSelectStatements();
         }
 
+        if (($hints[SqlWalker::HINT_PARTIAL] ?? false) === true || ($hints[Query::HINT_FORCE_PARTIAL_LOAD] ?? false) === true) {
+            throw FeatureNotImplemented::partialEntities();
+        }
+
         if (! ($key->cacheMode & Cache::MODE_PUT)) {
             return false;
         }

src/Cache/Exception/FeatureNotImplemented.php

@@ -20,4 +20,9 @@ public static function nonSelectStatements(): self
     {
         return new self('Second-level cache query supports only select statements.');
     }
+
+    public static function partialEntities(): self
+    {
+        return new self('Second level cache does not support partial entities.');
+    }
 }