Skip to content

Latest commit

 

History

History
239 lines (236 loc) · 8.91 KB

148.md

File metadata and controls

239 lines (236 loc) · 8.91 KB

Results of GPT-4 on HumanEval/148

Date: 18 July, 2023
Model: gpt-4
Temperature: 0.2
Passing runs: 8
Total runs: 10
Success rate: 80%

Prompt

Tokens: 237

def bf(planet1, planet2):
    '''
    There are eight planets in our solar system: the closerst to the Sun 
    is Mercury, the next one is Venus, then Earth, Mars, Jupiter, Saturn, 
    Uranus, Neptune.
    Write a function that takes two planet names as strings planet1 and planet2. 
    The function should return a tuple containing all planets whose orbits are 
    located between the orbit of planet1 and the orbit of planet2, sorted by 
    the proximity to the sun. 
    The function should return an empty tuple if planet1 or planet2
    are not correct planet names. 
    Examples
    bf("Jupiter", "Neptune") ==> ("Saturn", "Uranus")
    bf("Earth", "Mercury") ==> ("Venus")
    bf("Mercury", "Uranus") ==> ("Venus", "Earth", "Mars", "Jupiter", "Saturn")
    '''

The above is an incomplete Python code fragment. Return the complete and correct code with no additional text.

Correct solution

def bf(planet1, planet2):
    '''
    There are eight planets in our solar system: the closerst to the Sun 
    is Mercury, the next one is Venus, then Earth, Mars, Jupiter, Saturn, 
    Uranus, Neptune.
    Write a function that takes two planet names as strings planet1 and planet2. 
    The function should return a tuple containing all planets whose orbits are 
    located between the orbit of planet1 and the orbit of planet2, sorted by 
    the proximity to the sun. 
    The function should return an empty tuple if planet1 or planet2
    are not correct planet names. 
    Examples
    bf("Jupiter", "Neptune") ==> ("Saturn", "Uranus")
    bf("Earth", "Mercury") ==> ("Venus")
    bf("Mercury", "Uranus") ==> ("Venus", "Earth", "Mars", "Jupiter", "Saturn")
    '''planet_names = ("Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune")
    if planet1 not in planet_names or planet2 not in planet_names or planet1 == planet2:
        return ()
    planet1_index = planet_names.index(planet1)
    planet2_index = planet_names.index(planet2)
    if planet1_index < planet2_index:
        return (planet_names[planet1_index + 1: planet2_index])
    else:
        return (planet_names[planet2_index + 1 : planet1_index])

Unit tests

def check(candidate):

    # Check some simple cases
    assert candidate("Jupiter", "Neptune") == ("Saturn", "Uranus"), "First test error: " + str(len(candidate("Jupiter", "Neptune")))      
    assert candidate("Earth", "Mercury") == ("Venus",), "Second test error: " + str(candidate("Earth", "Mercury"))  
    assert candidate("Mercury", "Uranus") == ("Venus", "Earth", "Mars", "Jupiter", "Saturn"), "Third test error: " + str(candidate("Mercury", "Uranus"))      
    assert candidate("Neptune", "Venus") == ("Earth", "Mars", "Jupiter", "Saturn", "Uranus"), "Fourth test error: " + str(candidate("Neptune", "Venus"))  


    # Check some edge cases that are easy to work out by hand.
    assert candidate("Earth", "Earth") == ()
    assert candidate("Mars", "Earth") == ()
    assert candidate("Jupiter", "Makemake") == ()

Runs

✅ Run 1

Start time: 2023-07-18 17:50:24
Duration: 10.81s
Completion tokens: 112
Result: Pass

def bf(planet1, planet2):
    planets = ["Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune"]
    if planet1 not in planets or planet2 not in planets:
        return ()
    else:
        start = min(planets.index(planet1), planets.index(planet2))
        end = max(planets.index(planet1), planets.index(planet2))
        return tuple(planets[start+1:end])

✅ Run 2

Start time: 2023-07-18 17:50:15
Duration: 8.55s
Completion tokens: 112
Result: Pass

def bf(planet1, planet2):
    planets = ["Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune"]
    if planet1 not in planets or planet2 not in planets:
        return ()
    else:
        start = min(planets.index(planet1), planets.index(planet2))
        end = max(planets.index(planet1), planets.index(planet2))
        return tuple(planets[start+1:end])

✅ Run 3

Start time: 2023-07-18 17:50:06
Duration: 8.9s
Completion tokens: 112
Result: Pass

def bf(planet1, planet2):
    planets = ["Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune"]
    if planet1 not in planets or planet2 not in planets:
        return ()
    else:
        start = min(planets.index(planet1), planets.index(planet2))
        end = max(planets.index(planet1), planets.index(planet2))
        return tuple(planets[start+1:end])

✅ Run 4

Start time: 2023-07-18 17:49:55
Duration: 10.87s
Completion tokens: 112
Result: Pass

def bf(planet1, planet2):
    planets = ["Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune"]
    if planet1 not in planets or planet2 not in planets:
        return ()
    else:
        start = min(planets.index(planet1), planets.index(planet2))
        end = max(planets.index(planet1), planets.index(planet2))
        return tuple(planets[start+1:end])

✅ Run 5

Start time: 2023-07-18 17:49:45
Duration: 10.59s
Completion tokens: 112
Result: Pass

def bf(planet1, planet2):
    planets = ["Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune"]
    if planet1 not in planets or planet2 not in planets:
        return ()
    else:
        start = min(planets.index(planet1), planets.index(planet2))
        end = max(planets.index(planet1), planets.index(planet2))
        return tuple(planets[start+1:end])

✅ Run 6

Start time: 2023-07-18 17:49:35
Duration: 9.22s
Completion tokens: 112
Result: Pass

def bf(planet1, planet2):
    planets = ["Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune"]
    if planet1 not in planets or planet2 not in planets:
        return ()
    else:
        start = min(planets.index(planet1), planets.index(planet2))
        end = max(planets.index(planet1), planets.index(planet2))
        return tuple(planets[start+1:end])

✅ Run 7

Start time: 2023-07-18 17:49:11
Duration: 23.9s
Completion tokens: 308
Result: Pass

def bf(planet1, planet2):
    '''
    There are eight planets in our solar system: the closerst to the Sun 
    is Mercury, the next one is Venus, then Earth, Mars, Jupiter, Saturn, 
    Uranus, Neptune.
    Write a function that takes two planet names as strings planet1 and planet2. 
    The function should return a tuple containing all planets whose orbits are 
    located between the orbit of planet1 and the orbit of planet2, sorted by 
    the proximity to the sun. 
    The function should return an empty tuple if planet1 or planet2
    are not correct planet names. 
    Examples
    bf("Jupiter", "Neptune") ==> ("Saturn", "Uranus")
    bf("Earth", "Mercury") ==> ("Venus")
    bf("Mercury", "Uranus") ==> ("Venus", "Earth", "Mars", "Jupiter", "Saturn")
    '''
    planets = ["Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune"]
    if planet1 not in planets or planet2 not in planets:
        return ()
    else:
        start = min(planets.index(planet1), planets.index(planet2))
        end = max(planets.index(planet1), planets.index(planet2))
        return tuple(planets[start+1:end])

✅ Run 8

Start time: 2023-07-18 17:49:02
Duration: 9.6s
Completion tokens: 112
Result: Pass

def bf(planet1, planet2):
    planets = ["Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune"]
    if planet1 not in planets or planet2 not in planets:
        return ()
    else:
        start = min(planets.index(planet1), planets.index(planet2))
        end = max(planets.index(planet1), planets.index(planet2))
        return tuple(planets[start+1:end])

❌ Run 9

Start time: 2023-07-18 17:48:52
Duration: 9.77s
Completion tokens: 112
Result: Fail

```python
def bf(planet1, planet2):
    planets = ["Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune"]
    if planet1 not in planets or planet2 not in planets:
        return ()
    else:
        start = min(planets.index(planet1), planets.index(planet2))
        end = max(planets.index(planet1), planets.index(planet2))
        return tuple(planets[start+1:end])```

❌ Run 10

Start time: 2023-07-18 17:48:41
Duration: 10.73s
Completion tokens: 112
Result: Fail

```python
def bf(planet1, planet2):
    planets = ["Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune"]
    if planet1 not in planets or planet2 not in planets:
        return ()
    else:
        start = min(planets.index(planet1), planets.index(planet2))
        end = max(planets.index(planet1), planets.index(planet2))
        return tuple(planets[start+1:end])```