001 /*
002 * Copyright 2011 Christian Kumpe http://kumpe.de/christian/java
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016 package de.kumpe.hadooptimizer.examples;
017
018 import java.util.Date;
019
020 import org.apache.commons.cli.Option;
021 import org.apache.commons.cli.OptionGroup;
022 import org.apache.commons.cli.Options;
023 import org.apache.commons.cli.ParseException;
024 import org.apache.commons.cli.PatternOptionBuilder;
025 import org.apache.hadoop.conf.Configurable;
026
027 import de.kumpe.hadooptimizer.EaOptimizerConfiguration;
028 import de.kumpe.hadooptimizer.EaOptimizerConfigurationBase;
029 import de.kumpe.hadooptimizer.EsIndividual;
030 import de.kumpe.hadooptimizer.EsOptimizerConfiguration;
031 import de.kumpe.hadooptimizer.EsWrappableMutator;
032 import de.kumpe.hadooptimizer.Halter;
033 import de.kumpe.hadooptimizer.Mutator;
034 import de.kumpe.hadooptimizer.Optimizer;
035 import de.kumpe.hadooptimizer.Recombiner;
036 import de.kumpe.hadooptimizer.hadoop.EaHadoOptimizer;
037 import de.kumpe.hadooptimizer.hadoop.EsHadoOptimizer;
038 import de.kumpe.hadooptimizer.hadoop.EsMultiPopulationsHadoOptimizer;
039 import de.kumpe.hadooptimizer.impl.ComputingTimeLimitHalter;
040 import de.kumpe.hadooptimizer.impl.CountingHalter;
041 import de.kumpe.hadooptimizer.impl.DuplicatingRecombiner;
042 import de.kumpe.hadooptimizer.impl.EsEvaluatorWrapper;
043 import de.kumpe.hadooptimizer.impl.EsHalterWrapper;
044 import de.kumpe.hadooptimizer.impl.EsMutatorWrapper;
045 import de.kumpe.hadooptimizer.impl.EsPopulationReaderWrapper;
046 import de.kumpe.hadooptimizer.impl.EsPopulationWriterWrapper;
047 import de.kumpe.hadooptimizer.impl.EvaluationLimitHalter;
048 import de.kumpe.hadooptimizer.impl.LoggingReporter;
049 import de.kumpe.hadooptimizer.impl.MemoryPopulationReader;
050 import de.kumpe.hadooptimizer.impl.NullPopulationWriter;
051 import de.kumpe.hadooptimizer.impl.ReportingHalterWrapper;
052 import de.kumpe.hadooptimizer.impl.ReportingHalterWrapper.Reporter;
053 import de.kumpe.hadooptimizer.impl.TextFileReporter;
054 import de.kumpe.hadooptimizer.impl.TimeBasedHalter;
055 import de.kumpe.hadooptimizer.simple.SimpleEaOptimizer;
056 import de.kumpe.hadooptimizer.simple.SimpleEsOptimizer;
057 import de.kumpe.hadooptimizer.simple.ThreadedEaOptimizer;
058 import de.kumpe.hadooptimizer.simple.ThreadedEsOptimizer;
059
060 /**
061 * Base class for all "executables" in the examples module. It delivers a basic
062 * infrastructure for a common workflow:
063 * <ol>
064 * <li>{@link #createOptions()}
065 * <li>{@link #processCommandLine()}
066 * <li>{@link #execute()}
067 * </ol>
068 *
069 * @param <I>
070 * the individual's type
071 *
072 * @author <a href="http://kumpe.de/christian/java">Christian Kumpe</a>
073 */
074 public abstract class OptimizerExample<I> extends Example {
075 protected static final String OPTION_STANDARD_DEVIATION = "d";
076 protected static final String OPTION_ZERO_PADDING = "z";
077 protected static final String OPTION_MULTI_POPULATIONS = "m";
078 protected static final String OPTION_ES = "es";
079 protected static final String OPTION_EA = "ea";
080 protected static final String OPTION_OFFSPRING = "o";
081 protected static final String OPTION_PRESERVE_PARENTS = "k";
082 protected static final String OPTION_EVALUATION_LIMIT = "limit";
083 protected static final String OPTION_TIME_LIMIT = "time";
084 protected static final String OPTION_NS_LIMIT = "ns";
085 protected static final String OPTION_CYCLES = "n";
086 protected static final String OPTION_PARENTS = "p";
087
088 protected int parents;
089 protected boolean preserveParents;
090 protected int offspring;
091 protected double standardDeviation = 1d;
092 private ReportingHalterWrapper<I> reportingHalterWrapper;
093
094 @Override
095 protected Options createOptions() throws Exception {
096 final Options options = super.createOptions();
097
098 final OptionGroup haltingGroup = new OptionGroup();
099 haltingGroup.setRequired(true);
100
101 final Option cyclesOption = new Option(OPTION_CYCLES, true,
102 "limit number of cycles");
103 cyclesOption.setArgName("nrOfCycles");
104 cyclesOption.setType(PatternOptionBuilder.NUMBER_VALUE);
105 haltingGroup.addOption(cyclesOption);
106
107 final Option nsLimitOption = new Option(OPTION_NS_LIMIT, true,
108 "limit computing time by number of ns");
109 nsLimitOption.setArgName("nrOfNs");
110 nsLimitOption.setType(PatternOptionBuilder.NUMBER_VALUE);
111 haltingGroup.addOption(nsLimitOption);
112
113 final Option timeLimitOption = new Option(OPTION_TIME_LIMIT, true,
114 "limit computing time until specified date and time");
115 timeLimitOption.setArgName("dateTime");
116 timeLimitOption.setType(PatternOptionBuilder.DATE_VALUE);
117 haltingGroup.addOption(timeLimitOption);
118
119 final Option evaluationLimitOption = new Option(
120 OPTION_EVALUATION_LIMIT, true,
121 "optimize until the specified evaluation is reached");
122 evaluationLimitOption.setArgName("dateTime");
123 evaluationLimitOption.setType(PatternOptionBuilder.NUMBER_VALUE);
124 haltingGroup.addOption(evaluationLimitOption);
125
126 options.addOptionGroup(haltingGroup);
127
128 final Option parentsOption = new Option(OPTION_PARENTS, "parents",
129 true, "number of parents");
130 parentsOption.setArgName("nrOfParents");
131 parentsOption.setType(PatternOptionBuilder.NUMBER_VALUE);
132 options.addOption(parentsOption);
133
134 final Option preserveParentsOption = new Option(
135 OPTION_PRESERVE_PARENTS, "keepParents", true,
136 "whether to keep parents");
137 preserveParentsOption.setOptionalArg(true);
138 preserveParentsOption.setArgName("true|false");
139 options.addOption(preserveParentsOption);
140
141 final Option offspringOption = new Option(OPTION_OFFSPRING,
142 "offspring", true, "number of offspring");
143 offspringOption.setArgName("nrOfOffspring");
144 offspringOption.setType(PatternOptionBuilder.NUMBER_VALUE);
145 options.addOption(offspringOption);
146
147 final OptionGroup typeOptionGroup = new OptionGroup();
148
149 final Option eaOption = new Option(OPTION_EA, true,
150 "use an EA-optimizer of the sepcified type");
151 eaOption.setOptionalArg(true);
152 eaOption.setArgName("simple|threaded|hadoop");
153 typeOptionGroup.addOption(eaOption);
154
155 final Option esOption = new Option(OPTION_ES, true,
156 "use an ES-optimizer of the sepcified type");
157 esOption.setOptionalArg(true);
158 esOption.setArgName("simple|threaded|hadoop");
159 typeOptionGroup.addOption(esOption);
160
161 options.addOptionGroup(typeOptionGroup);
162
163 final Option multiPopulationsOption = new Option(
164 OPTION_MULTI_POPULATIONS,
165 "multiPopulation",
166 true,
167 "use the multi-population optimizer with the given number of inner cycles; only valid with -es hadoop");
168 multiPopulationsOption.setOptionalArg(true);
169 multiPopulationsOption.setArgName("nrOfInnerCycles");
170 multiPopulationsOption.setType(PatternOptionBuilder.NUMBER_VALUE);
171 options.addOption(multiPopulationsOption);
172
173 final Option zeroPaddingOption = new Option(
174 OPTION_ZERO_PADDING,
175 "zeroPadding",
176 false,
177 "when an individual is given on the command line, fill missing elements with zeros");
178 options.addOption(zeroPaddingOption);
179
180 final Option standardDeviationOption = new Option(
181 OPTION_STANDARD_DEVIATION, "standardDeviation", true,
182 "the standard deviation of a gaussion mutator");
183 standardDeviationOption.setArgName("standardDeviation");
184 standardDeviationOption.setType(PatternOptionBuilder.NUMBER_VALUE);
185 options.addOption(standardDeviationOption);
186
187 return options;
188 }
189
190 @Override
191 protected void processCommandLine() throws Exception {
192 super.processCommandLine();
193
194 final Number parents = (Long) commandLine
195 .getParsedOptionValue(OPTION_PARENTS);
196 if (null != parents) {
197 this.parents = parents.intValue();
198
199 }
200
201 if (commandLine.hasOption(OPTION_PRESERVE_PARENTS)) {
202 preserveParents = Boolean.parseBoolean(commandLine.getOptionValue(
203 OPTION_PRESERVE_PARENTS, "true"));
204 }
205
206 final Number offspring = (Long) commandLine
207 .getParsedOptionValue(OPTION_OFFSPRING);
208 if (null != offspring) {
209 this.offspring = offspring.intValue();
210 }
211
212 final Number standardDeviation = (Number) commandLine
213 .getParsedOptionValue(OPTION_STANDARD_DEVIATION);
214 if (null != standardDeviation) {
215 this.standardDeviation = standardDeviation.doubleValue();
216 }
217 }
218
219 @Override
220 protected void execute() throws Exception {
221 final Optimizer<?> optimizer;
222 if (commandLine.hasOption(OPTION_EA)) {
223 optimizer = createEaOptimizer();
224 } else {
225 optimizer = createEsOptimizer();
226 }
227
228 if (optimizer instanceof Configurable) {
229 ((Configurable) optimizer).setConf(getConf());
230 }
231
232 optimizer.optimize();
233 }
234
235 protected EaOptimizerConfiguration<I> createEaOptimizerConfiguration()
236 throws ParseException {
237 throw new UnsupportedOperationException(
238 "this example does not support EA-configurations");
239 }
240
241 protected EsOptimizerConfiguration createEsOptimizerConfiguration()
242 throws ParseException {
243 @SuppressWarnings("unchecked")
244 final EaOptimizerConfiguration<double[]> eaConf = (EaOptimizerConfiguration<double[]>) createEaOptimizerConfiguration();
245 eaConf.validate();
246 final EsOptimizerConfiguration esConf = new EsOptimizerConfiguration();
247 esConf.setPopulationReader(new EsPopulationReaderWrapper(eaConf
248 .getPopulationReader()));
249 esConf.setPopulationWriter(new EsPopulationWriterWrapper(eaConf
250 .getPopulationWriter()));
251 esConf.setHalter(new EsHalterWrapper(eaConf.getHalter()));
252 final Mutator<double[]> delegatesMutator = eaConf.getMutator();
253 if (!(delegatesMutator instanceof EsWrappableMutator)) {
254 throw new UnsupportedOperationException(
255 "cannot wrap the delegate's Mutator: " + delegatesMutator);
256 }
257 esConf.setMutator(new EsMutatorWrapper(
258 (EsWrappableMutator) delegatesMutator));
259 esConf.setEvaluator(new EsEvaluatorWrapper(eaConf.getEvaluator()));
260 esConf.setParents(eaConf.getParents());
261 esConf.setPreserveParents(eaConf.isPreserveParents());
262
263 final Recombiner<double[]> delegatesPairingSelector = eaConf
264 .getRecombiner();
265 if (!(delegatesPairingSelector instanceof DuplicatingRecombiner)) {
266 throw new UnsupportedOperationException(
267 "cannot extract offspring from delegate's PairingSelector: "
268 + delegatesPairingSelector);
269 }
270 esConf.setOffspring(((DuplicatingRecombiner<double[]>) delegatesPairingSelector)
271 .getOffspring());
272
273 return esConf;
274 }
275
276 protected void initConfiguration(
277 final EaOptimizerConfigurationBase<I> configuration,
278 final I startIndividual) {
279 configuration.setPopulationReader(new MemoryPopulationReader<I>(
280 startIndividual));
281 configuration.setPopulationWriter(new NullPopulationWriter<I>());
282
283 final Halter<I> halter = createHalter();
284 reportingHalterWrapper = new ReportingHalterWrapper<I>(halter);
285 if (halter instanceof CountingHalter) {
286 addReporter(new LoggingReporter<I>((CountingHalter<I>) halter));
287 } else {
288 addReporter(new LoggingReporter<I>(null));
289 }
290
291 configuration.setHalter(reportingHalterWrapper);
292 addReporter(new TextFileReporter<I>(logFile));
293
294 configuration.setParents(parents);
295 configuration.setPreserveParents(preserveParents);
296 }
297
298 private Halter<I> createHalter() {
299 try {
300 final Long cycles = (Long) commandLine
301 .getParsedOptionValue(OPTION_CYCLES);
302 if (null != cycles) {
303 return new CountingHalter<I>(cycles.intValue());
304 }
305 final Long ns = (Long) commandLine
306 .getParsedOptionValue(OPTION_NS_LIMIT);
307 if (null != ns) {
308 return new ComputingTimeLimitHalter<I>(ns.longValue());
309 }
310 final Number limit = (Number) commandLine
311 .getParsedOptionValue(OPTION_EVALUATION_LIMIT);
312 if (null != limit) {
313 return new EvaluationLimitHalter<I>(limit.doubleValue());
314 }
315 final Date time = (Date) commandLine
316 .getParsedOptionValue(OPTION_TIME_LIMIT);
317 return new TimeBasedHalter<I>(time);
318 } catch (final ParseException e) {
319 throw new RuntimeException(e);
320 }
321 }
322
323 protected void initConfiguration(
324 final EaOptimizerConfiguration<I> configuration,
325 final I startIndividual) {
326 initConfiguration((EaOptimizerConfigurationBase<I>) configuration,
327 startIndividual);
328 configuration.setRecombiner(new DuplicatingRecombiner<I>(offspring));
329 }
330
331 @SuppressWarnings("unchecked")
332 protected void initConfiguration(
333 final EsOptimizerConfiguration configuration,
334 final I startIndividual) {
335 initConfiguration((EaOptimizerConfigurationBase<I>) configuration,
336 startIndividual);
337 configuration.setOffspring(offspring);
338 }
339
340 protected void addReporter(final Reporter<I> reporter) {
341 reportingHalterWrapper.addReporter(reporter);
342 }
343
344 private Optimizer<I> createEaOptimizer() throws ParseException {
345 checkForInvalidMultiplePopulationOption();
346
347 final EaOptimizerConfiguration<I> configuration = createEaOptimizerConfiguration();
348
349 final String type = commandLine.getOptionValue(OPTION_EA, "simple");
350 if ("simple".equalsIgnoreCase(type)) {
351 return new SimpleEaOptimizer<I>(configuration);
352 }
353 if ("threaded".equalsIgnoreCase(type)) {
354 return new ThreadedEaOptimizer<I>(configuration);
355 }
356 if ("hadoop".equalsIgnoreCase(type)) {
357 return new EaHadoOptimizer<I>(configuration);
358 }
359
360 throw new IllegalArgumentException("type not supported: " + type);
361 }
362
363 private Optimizer<EsIndividual> createEsOptimizer() throws ParseException {
364 final EsOptimizerConfiguration configuration = createEsOptimizerConfiguration();
365
366 final String type = commandLine.getOptionValue(OPTION_ES, "simple");
367 if ("hadoop".equalsIgnoreCase(type)) {
368 if (!commandLine.hasOption(OPTION_MULTI_POPULATIONS)) {
369 return new EsHadoOptimizer(configuration);
370 }
371
372 Long innerCycleCount = (Long) commandLine
373 .getParsedOptionValue(OPTION_MULTI_POPULATIONS);
374 if (null == innerCycleCount) {
375 innerCycleCount = 10l;
376 }
377 return createMultiPopulationsHadoopEsOptimizer(configuration,
378 innerCycleCount.intValue());
379 }
380
381 checkForInvalidMultiplePopulationOption();
382
383 if ("simple".equalsIgnoreCase(type)) {
384 return new SimpleEsOptimizer(configuration);
385 }
386 if ("threaded".equalsIgnoreCase(type)) {
387 return new ThreadedEsOptimizer(configuration);
388 }
389
390 throw new IllegalArgumentException("type not supported: " + type);
391 }
392
393 private Optimizer<EsIndividual> createMultiPopulationsHadoopEsOptimizer(
394 final EsOptimizerConfiguration configuration,
395 final int innerCycleCount) {
396 final Halter<EsIndividual> outerHalter = configuration.getHalter();
397 final CountingHalter<EsIndividual> countingHalter = new CountingHalter<EsIndividual>(
398 innerCycleCount);
399 final ReportingHalterWrapper<EsIndividual> reportingHalterWrapper = new ReportingHalterWrapper<EsIndividual>(
400 countingHalter);
401 reportingHalterWrapper.addReporter(new LoggingReporter<EsIndividual>(
402 countingHalter));
403 configuration.setHalter(reportingHalterWrapper);
404
405 return new EsMultiPopulationsHadoOptimizer(configuration, outerHalter);
406 }
407
408 private void checkForInvalidMultiplePopulationOption() {
409 if (commandLine.hasOption(OPTION_MULTI_POPULATIONS)) {
410 throw new IllegalArgumentException(
411 "multiPopulation is only allowed with ES-Optimizer of type hadoop");
412 }
413 }
414 }