- author = "Andrew Nader and Danielle Azar",
- title = "Searching for Activation Functions Using a Self-Adaptive Evolutionary Algorithm",
- year = "2020",
- editor = "Richard Allmendinger and Hugo Terashima Marin and Efren Mezura Montes and Thomas Bartz-Beielstein and Bogdan Filipic and Ke Tang and David Howard and Emma Hart and Gusz Eiben and Tome Eftimov and William {La Cava} and Boris Naujoks and Pietro Oliveto and Vanessa Volz and Thomas Weise and Bilel Derbel and Ke Li and Xiaodong Li and Saul Zapotecas and Qingfu Zhang and Rui Wang and Ran Cheng and Guohua Wu and Miqing Li and Hisao Ishibuchi and Jonathan Fieldsend and Ozgur Akman and Khulood Alyahya and Juergen Branke and John R. Woodward and Daniel R. Tauritz and Marco Baioletti and Josu Ceberio Uribe and John McCall and Alfredo Milani and Stefan Wagner and Michael Affenzeller and Bradley Alexander and Alexander (Sandy) Brownlee and Saemundur O. Haraldsson and Markus Wagner and Nayat Sanchez-Pi and Luis Marti and Silvino {Fernandez Alzueta} and Pablo {Valledor Pellicer} and Thomas Stuetzle and Matthew Johns and Nick Ross and Ed Keedwell and Herman Mahmoud and David Walker and Anthony Stein and Masaya Nakata and David Paetzel and Neil Vaughan and Stephen Smith and Stefano Cagnoni and Robert M. Patton and Ivanoe {De Falco} and Antonio {Della Cioppa} and Umberto Scafuri and Ernesto Tarantino and Akira Oyama and Koji Shimoyama and Hemant Kumar Singh and Kazuhisa Chiba and Pramudita Satria Palar and Alma Rahat and Richard Everson and Handing Wang and Yaochu Jin and Erik Hemberg and Riyad Alshammari and Tokunbo Makanju and Fuijimino-shi and Ivan Zelinka and Swagatam Das and Ponnuthurai Nagaratnam and Roman Senkerik",
- booktitle = "Proceedings of the 2020 Genetic and Evolutionary Computation Conference Companion",
- pages = "145--146",
- address = "internet",
- series = "GECCO '20",
- month = jul # " 8-12",
- organisation = "SIGEVO",
- publisher = "Association for Computing Machinery",
- publisher_address = "New York, NY, USA",
- keywords = "genetic algorithms, genetic programming, ANN, activation functions, deep learning, neural architecture search, evolutionary computation, self-adaptive genetic algorithms, neural networks",
- isbn13 = "9781450371278",
-
URL = "
http://www.human-competitive.org/sites/default/files/azar_entry_form.txt",
-
URL = "
http://www.human-competitive.org/sites/default/files/nader-azar-gecco2020.pdf",
-
URL = "https://doi.org/10.1145/3377929.3389942",
-
DOI = "
10.1145/3377929.3389942",
-
video_url = "http://www.human-competitive.org/sites/default/files/nader_azar_humies.mp4",
-
video_url = "https://youtu.be/MjV4kCGfERw",
- size = "2 pages",
- abstract = "The introduction of the ReLU function in neural network architectures yielded substantial improvements over sigmoidal activation functions and allowed for the training of deep networks. Ever since, the search for new activation functions in neural networks has been an active research topic. However, to the best of our knowledge, the design of new activation functions has mostly been done by hand. In this work, we propose the use of a self-adaptive evolutionary algorithm that searches for new activation functions using a genetic programming approach, and we compare the performance of the obtained activation functions to ReLU. We also analyze the shape of the obtained activations to see if they have any common traits such as monotonicity or piece-wise linearity, and we study the effects of the self-adaptation to see which operators perform well in the context of a search for new activation functions. We perform a thorough experimental study on datasets of different sizes and types, using different types of neural network architectures. We report favorable results obtained from the mean and standard deviation of the performance metrics over multiple runs.",
-
notes = "Entered 2021 HUMIES. Neural architecture (ANN) search,
NAS.
Also known as \cite{10.1145/3377929.3389942} GECCO-2020 A Recombination of the 29th International Conference on Genetic Algorithms (ICGA) and the 25th Annual Genetic Programming Conference (GP)",
