- author = "Vadim Liventsev and Anastasiia Grishina and Aki Harma and Leon Moonen",
- title = "Fully Autonomous Programming with Large Language Models",
- booktitle = "Proceedings of the 2023 Genetic and Evolutionary Computation Conference",
- year = "2023",
- editor = "Sara Silva and Luis Paquete and Leonardo Vanneschi and Nuno Lourenco and Ales Zamuda and Ahmed Kheiri and Arnaud Liefooghe and Bing Xue and Ying Bi and Nelishia Pillay and Irene Moser and Arthur Guijt and Jessica Catarino and Pablo Garcia-Sanchez and Leonardo Trujillo and Carla Silva and Nadarajen Veerapen",
- pages = "1146--1155",
- address = "Lisbon, Portugal",
- series = "GECCO '23",
- month = "15-19 " # jul,
- organisation = "SIGEVO",
- publisher = "Association for Computing Machinery",
- publisher_address = "New York, NY, USA",
- keywords = "genetic algorithms, genetic programming, genetic improvement, APR, SBSE, software design engineering, Computing methodologies, ANN, Model development and analysis, Search methodologies, automatic programming, large language models, program repair",
- isbn13 = "979-8-4007-0119-1/23/07",
-
URL = "
https://human-competitive.org/sites/default/files/grishinaentry.txt",
-
URL = "
https://human-competitive.org/sites/default/files/fullyautonomousprogrammingwithlargelanguagemodels.pdf",
-
DOI = "
10.1145/3583131.3590481",
- size = "10 pages",
- abstract = "Current approaches to program synthesis with Large Language Models (LLMs) exhibit a near miss syndrome: they tend to generate programs that semantically resemble the correct answer (as measured by text similarity metrics or human evaluation), but achieve a low or even zero accuracy as measured by unit tests due to small imperfections, such as the wrong input or output format. This calls for an approach known as Synthesize, Execute, Debug (SED), whereby a draft of the solution is generated first, followed by a program repair phase addressing the failed tests. To effectively apply this approach to instruction-driven LLMs, one needs to determine which prompts perform best as instructions for LLMs, as well as strike a balance between repairing unsuccessful programs and replacing them with newly generated ones. We explore these trade-offs empirically, comparing replace-focused, repair-focused, and hybrid debug strategies, as well as different template-based and model-bas",
-
notes = "Finalist 2023 HUMIES
GECCO-2023 A Recombination of the 32nd International Conference on Genetic Algorithms (ICGA) and the 28th Annual Genetic Programming Conference (GP)",
