Cannot query derived fields directlly by defining them on search request on OS-2.15

[Minhal-111]

Description: As mentioned in the document, you can define derived fields directly in a search request and query them along with regular indexed fields.

However when I query derived fields without defining them in my index mappings , I won't retrieve any fields.

To Reproduce
Create an index "logs"

PUT logs
{
  "mappings": {
    "properties": {
      "request": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        }
      },
      "client_ip": {
        "type": "keyword"
      }
    }
  }
}

POST sample data:

POST _bulk
{ "index" : { "_index" : "logs", "_id" : "1" } }
{ "request": "894030400 GET /english/images/france98_venues.gif HTTP/1.0 200 778", "clientip": "61.177.2.0" }
{ "index" : { "_index" : "logs", "_id" : "2" } }
{ "request": "894140400 GET /french/playing/mascot/mascot.html HTTP/1.1 200 5474", "clientip": "185.92.2.0" }
{ "index" : { "_index" : "logs", "_id" : "3" } }
{ "request": "894250400 POST /english/venues/images/venue_header.gif HTTP/1.0 200 711", "clientip": "61.177.2.0" }
{ "index" : { "_index" : "logs", "_id" : "4" } }
{ "request": "894360400 POST /images/home_fr_button.gif HTTP/1.1 200 2140", "clientip": "129.178.2.0" }
{ "index" : { "_index" : "logs", "_id" : "5" } }
{ "request": "894470400 DELETE /images/102384s.gif HTTP/1.0 200 785", "clientip": "227.177.2.0" }

Search derived fields in a search request:

POST /logs/_search
{
  "derived": {
    "url": {
      "type": "text",
      "script": {
        "source": """emit(doc["request"].value.splitOnToken(" ")[2])"""
      }
    },
    "status": {
      "type": "keyword",
      "script": {
        "source": """emit(doc["request"].value.splitOnToken(" ")[4])"""
      }
    }
  },
  "query": {
    "match_all": {}
  },
  "fields": ["request", "clientip", "url", "status"]
}

Output:

{
  "took": 34,
  "timed_out": false,
  "_shards": {
    "total": 5,
    "successful": 5,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 5,
      "relation": "eq"
    },
    "max_score": 1,
    "hits": [
      {
        "_index": "logs",
        "_id": "3",
        "_score": 1,
        "_source": {
          "request": "894250400 POST /english/venues/images/venue_header.gif HTTP/1.0 200 711",
          "clientip": "61.177.2.0"
        },
        "fields": {
          "request": [
            "894250400 POST /english/venues/images/venue_header.gif HTTP/1.0 200 711"
          ],
          "clientip": [
            "61.177.2.0"
          ]
        }
      },
      {
        "_index": "logs",
        "_id": "5",
        "_score": 1,
        "_source": {
          "request": "894470400 DELETE /images/102384s.gif HTTP/1.0 200 785",
          "clientip": "227.177.2.0"
        },
        "fields": {
          "request": [
            "894470400 DELETE /images/102384s.gif HTTP/1.0 200 785"
          ],
          "clientip": [
            "227.177.2.0"
          ]
        }
      },
      {
        "_index": "logs",
        "_id": "4",
        "_score": 1,
        "_source": {
          "request": "894360400 POST /images/home_fr_button.gif HTTP/1.1 200 2140",
          "clientip": "129.178.2.0"
        },
        "fields": {
          "request": [
            "894360400 POST /images/home_fr_button.gif HTTP/1.1 200 2140"
          ],
          "clientip": [
            "129.178.2.0"
          ]
        }
      },
      {
        "_index": "logs",
        "_id": "2",
        "_score": 1,
        "_source": {
          "request": "894140400 GET /french/playing/mascot/mascot.html HTTP/1.1 200 5474",
          "clientip": "185.92.2.0"
        },
        "fields": {
          "request": [
            "894140400 GET /french/playing/mascot/mascot.html HTTP/1.1 200 5474"
          ],
          "clientip": [
            "185.92.2.0"
          ]
        }
      },
      {
        "_index": "logs",
        "_id": "1",
        "_score": 1,
        "_source": {
          "request": "894030400 GET /english/images/france98_venues.gif HTTP/1.0 200 778",
          "clientip": "61.177.2.0"
        },
        "fields": {
          "request": [
            "894030400 GET /english/images/france98_venues.gif HTTP/1.0 200 778"
          ],
          "clientip": [
            "61.177.2.0"
          ]
        }
      }
    ]
  }
}

Now defining derived fields in index mappings:

PUT /logs/_mapping
{
  "derived": {
    "url": {
      "type": "text",
      "script": {
        "source": """emit(doc["request.keyword"].value.splitOnToken(" ")[2])"""
      }
    },
    "status": {
      "type": "keyword",
      "script": {
        "source": """emit(doc["request.keyword"].value.splitOnToken(" ")[4])"""
      }
    }
  }
}

we get expected output as mentioned in doc:

{
  "took": 38,
  "timed_out": false,
  "_shards": {
    "total": 5,
    "successful": 5,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 5,
      "relation": "eq"
    },
    "max_score": 1,
    "hits": [
      {
        "_index": "logs",
        "_id": "3",
        "_score": 1,
        "_source": {
          "request": "894250400 POST /english/venues/images/venue_header.gif HTTP/1.0 200 711",
          "clientip": "61.177.2.0"
        },
        "fields": {
          "request": [
            "894250400 POST /english/venues/images/venue_header.gif HTTP/1.0 200 711"
          ],
          "url": [
            "/english/venues/images/venue_header.gif"
          ],
          "clientip": [
            "61.177.2.0"
          ],
          "status": [
            "200"
          ]
        }
      },
      {
        "_index": "logs",
        "_id": "5",
        "_score": 1,
        "_source": {
          "request": "894470400 DELETE /images/102384s.gif HTTP/1.0 200 785",
          "clientip": "227.177.2.0"
        },
        "fields": {
          "request": [
            "894470400 DELETE /images/102384s.gif HTTP/1.0 200 785"
          ],
          "url": [
            "/images/102384s.gif"
          ],
          "clientip": [
            "227.177.2.0"
          ],
          "status": [
            "200"
          ]
        }
      },
      {
        "_index": "logs",
        "_id": "4",
        "_score": 1,
        "_source": {
          "request": "894360400 POST /images/home_fr_button.gif HTTP/1.1 200 2140",
          "clientip": "129.178.2.0"
        },
        "fields": {
          "request": [
            "894360400 POST /images/home_fr_button.gif HTTP/1.1 200 2140"
          ],
          "url": [
            "/images/home_fr_button.gif"
          ],
          "clientip": [
            "129.178.2.0"
          ],
          "status": [
            "200"
          ]
        }
      },
      {
        "_index": "logs",
        "_id": "2",
        "_score": 1,
        "_source": {
          "request": "894140400 GET /french/playing/mascot/mascot.html HTTP/1.1 200 5474",
          "clientip": "185.92.2.0"
        },
        "fields": {
          "request": [
            "894140400 GET /french/playing/mascot/mascot.html HTTP/1.1 200 5474"
          ],
          "url": [
            "/french/playing/mascot/mascot.html"
          ],
          "clientip": [
            "185.92.2.0"
          ],
          "status": [
            "200"
          ]
        }
      },
      {
        "_index": "logs",
        "_id": "1",
        "_score": 1,
        "_source": {
          "request": "894030400 GET /english/images/france98_venues.gif HTTP/1.0 200 778",
          "clientip": "61.177.2.0"
        },
        "fields": {
          "request": [
            "894030400 GET /english/images/france98_venues.gif HTTP/1.0 200 778"
          ],
          "url": [
            "/english/images/france98_venues.gif"
          ],
          "clientip": [
            "61.177.2.0"
          ],
          "status": [
            "200"
          ]
        }
      }
    ]
  }
}