- author = "Dominik Sobania and Justyna Petke and Martin Briesch and Franz Rothlauf",
- title = "A Comparison of Large Language Models and Genetic Programming for Program Synthesis",
- journal = "IEEE Transactions on Evolutionary Computation",
- note = "Accepted for future publication",
- keywords = "genetic algorithms, genetic programming, PushGP, G3P, lexicase, SBSE, LLM, ANN, Software development management, Benchmark testing, Source coding, Codes, Software, Transformers, Software measurement, Program Synthesis, Large Language Models, Codex, GitHub Copilot, Software Engineering",
- ISSN = "1089-778X",
-
URL = "
https://discovery.ucl.ac.uk/id/eprint/10192702/1/bare_jrnl_new_sample4.pdf",
-
DOI = "
doi:10.1109/TEVC.2024.3410873",
-
code_url = "
https://github.com/domsob/github-copilot-generated-programs-2023/",
- size = "15 pages",
- abstract = "Large language models have recently become known for their ability to generate computer programs, especially through tools such as GitHub Copilot, a domain where genetic programming has been very successful so far. Although they require different inputs (free-text vs. input/output examples) their goal is the same: program synthesis. we compare how well GitHub Copilot and genetic programming perform on common program synthesis benchmark problems. We study the structure and diversity of the generated programs by using well-known software metrics. We find that GitHub Copilot and genetic programming solve a similar number of benchmark problems (85.2 percent vs. 77.8 percent, respectively). We find that GitHub Copilot generated smaller and less complex programs as genetic programming, while genetic programming is able to find new and unique problem solving strategies. This increase in diversity of solutions comes at a cost. When analysing the success rates for 100 runs per problem, GitHub Copilot outperforms genetic programming on over 50percent of the problems.",
-
notes = "also known as \cite{10551744}
benchmark problems from PSB1 and PSB2
G3P cites \cite{Forstenlechner:2018:PPSN}
lexicase \cite{Helmuth:2022:ALife}",
