001/*
002 * Copyright 2010-2016 UnboundID Corp.
003 * All Rights Reserved.
004 */
005/*
006 * Copyright (C) 2010-2016 UnboundID Corp.
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021package com.unboundid.ldap.sdk.examples;
022
023
024
025import java.io.IOException;
026import java.io.OutputStream;
027import java.io.Serializable;
028import java.text.ParseException;
029import java.util.LinkedHashMap;
030import java.util.LinkedHashSet;
031import java.util.List;
032import java.util.Random;
033import java.util.concurrent.CyclicBarrier;
034import java.util.concurrent.atomic.AtomicBoolean;
035import java.util.concurrent.atomic.AtomicLong;
036
037import com.unboundid.ldap.sdk.LDAPConnection;
038import com.unboundid.ldap.sdk.LDAPConnectionOptions;
039import com.unboundid.ldap.sdk.LDAPException;
040import com.unboundid.ldap.sdk.ResultCode;
041import com.unboundid.ldap.sdk.SearchScope;
042import com.unboundid.ldap.sdk.Version;
043import com.unboundid.util.ColumnFormatter;
044import com.unboundid.util.FixedRateBarrier;
045import com.unboundid.util.FormattableColumn;
046import com.unboundid.util.HorizontalAlignment;
047import com.unboundid.util.LDAPCommandLineTool;
048import com.unboundid.util.ObjectPair;
049import com.unboundid.util.OutputFormat;
050import com.unboundid.util.RateAdjustor;
051import com.unboundid.util.ResultCodeCounter;
052import com.unboundid.util.ThreadSafety;
053import com.unboundid.util.ThreadSafetyLevel;
054import com.unboundid.util.ValuePattern;
055import com.unboundid.util.WakeableSleeper;
056import com.unboundid.util.args.ArgumentException;
057import com.unboundid.util.args.ArgumentParser;
058import com.unboundid.util.args.BooleanArgument;
059import com.unboundid.util.args.FileArgument;
060import com.unboundid.util.args.IntegerArgument;
061import com.unboundid.util.args.ScopeArgument;
062import com.unboundid.util.args.StringArgument;
063
064import static com.unboundid.util.Debug.*;
065import static com.unboundid.util.StaticUtils.*;
066
067
068
069/**
070 * This class provides a tool that can be used to search an LDAP directory
071 * server repeatedly using multiple threads, and then modify each entry
072 * returned by that server.  It can help provide an estimate of the combined
073 * search and modify performance that a directory server is able to achieve.
074 * Either or both of the base DN and the search filter may be a value pattern as
075 * described in the {@link ValuePattern} class.  This makes it possible to
076 * search over a range of entries rather than repeatedly performing searches
077 * with the same base DN and filter.
078 * <BR><BR>
079 * Some of the APIs demonstrated by this example include:
080 * <UL>
081 *   <LI>Argument Parsing (from the {@code com.unboundid.util.args}
082 *       package)</LI>
083 *   <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util}
084 *       package)</LI>
085 *   <LI>LDAP Communication (from the {@code com.unboundid.ldap.sdk}
086 *       package)</LI>
087 *   <LI>Value Patterns (from the {@code com.unboundid.util} package)</LI>
088 * </UL>
089 * <BR><BR>
090 * All of the necessary information is provided using command line arguments.
091 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
092 * class, as well as the following additional arguments:
093 * <UL>
094 *   <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use
095 *       for the searches.  This must be provided.  It may be a simple DN, or it
096 *       may be a value pattern to express a range of base DNs.</LI>
097 *   <LI>"-s {scope}" or "--scope {scope}" -- specifies the scope to use for the
098 *       search.  The scope value should be one of "base", "one", "sub", or
099 *       "subord".  If this isn't specified, then a scope of "sub" will be
100 *       used.</LI>
101 *   <LI>"-f {filter}" or "--filter {filter}" -- specifies the filter to use for
102 *       the searches.  This must be provided.  It may be a simple filter, or it
103 *       may be a value pattern to express a range of filters.</LI>
104 *   <LI>"-A {name}" or "--attribute {name}" -- specifies the name of an
105 *       attribute that should be included in entries returned from the server.
106 *       If this is not provided, then all user attributes will be requested.
107 *       This may include special tokens that the server may interpret, like
108 *       "1.1" to indicate that no attributes should be returned, "*", for all
109 *       user attributes, or "+" for all operational attributes.  Multiple
110 *       attributes may be requested with multiple instances of this
111 *       argument.</LI>
112 *   <LI>"-m {name}" or "--modifyAttribute {name}" -- specifies the name of the
113 *       attribute to modify.  Multiple attributes may be modified by providing
114 *       multiple instances of this argument.  At least one attribute must be
115 *       provided.</LI>
116 *   <LI>"-l {num}" or "--valueLength {num}" -- specifies the length in bytes to
117 *       use for the values of the target attributes to modify.  If this is not
118 *       provided, then a default length of 10 bytes will be used.</LI>
119 *   <LI>"-C {chars}" or "--characterSet {chars}" -- specifies the set of
120 *       characters that will be used to generate the values to use for the
121 *       target attributes to modify.  It should only include ASCII characters.
122 *       Values will be generated from randomly-selected characters from this
123 *       set.  If this is not provided, then a default set of lowercase
124 *       alphabetic characters will be used.</LI>
125 *   <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of
126 *       concurrent threads to use when performing the searches.  If this is not
127 *       provided, then a default of one thread will be used.</LI>
128 *   <LI>"-i {sec}" or "--intervalDuration {sec}" -- specifies the length of
129 *       time in seconds between lines out output.  If this is not provided,
130 *       then a default interval duration of five seconds will be used.</LI>
131 *   <LI>"-I {num}" or "--numIntervals {num}" -- specifies the maximum number of
132 *       intervals for which to run.  If this is not provided, then it will
133 *       run forever.</LI>
134 *   <LI>"--iterationsBeforeReconnect {num}" -- specifies the number of search
135 *       iterations that should be performed on a connection before that
136 *       connection is closed and replaced with a newly-established (and
137 *       authenticated, if appropriate) connection.</LI>
138 *   <LI>"-r {ops-per-second}" or "--ratePerSecond {ops-per-second}" --
139 *       specifies the target number of operations to perform per second.  Each
140 *       search and modify operation will be counted separately for this
141 *       purpose, so if a value of 1 is specified and a search returns two
142 *       entries, then a total of three seconds will be required (one for the
143 *       search and one for the modify for each entry).  It is still necessary
144 *       to specify a sufficient number of threads for achieving this rate.  If
145 *       this option is not provided, then the tool will run at the maximum rate
146 *       for the specified number of threads.</LI>
147 *   <LI>"--variableRateData {path}" -- specifies the path to a file containing
148 *       information needed to allow the tool to vary the target rate over time.
149 *       If this option is not provided, then the tool will either use a fixed
150 *       target rate as specified by the "--ratePerSecond" argument, or it will
151 *       run at the maximum rate.</LI>
152 *   <LI>"--generateSampleRateFile {path}" -- specifies the path to a file to
153 *       which sample data will be written illustrating and describing the
154 *       format of the file expected to be used in conjunction with the
155 *       "--variableRateData" argument.</LI>
156 *   <LI>"--warmUpIntervals {num}" -- specifies the number of intervals to
157 *       complete before beginning overall statistics collection.</LI>
158 *   <LI>"--timestampFormat {format}" -- specifies the format to use for
159 *       timestamps included before each output line.  The format may be one of
160 *       "none" (for no timestamps), "with-date" (to include both the date and
161 *       the time), or "without-date" (to include only time time).</LI>
162 *   <LI>"-Y {authzID}" or "--proxyAs {authzID}" -- Use the proxied
163 *       authorization v2 control to request that the operations be processed
164 *       using an alternate authorization identity.  In this case, the bind DN
165 *       should be that of a user that has permission to use this control.  The
166 *       authorization identity may be a value pattern.</LI>
167 *   <LI>"--suppressErrorResultCodes" -- Indicates that information about the
168 *       result codes for failed operations should not be displayed.</LI>
169 *   <LI>"-c" or "--csv" -- Generate output in CSV format rather than a
170 *       display-friendly format.</LI>
171 * </UL>
172 */
173@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
174public final class SearchAndModRate
175       extends LDAPCommandLineTool
176       implements Serializable
177{
178  /**
179   * The serial version UID for this serializable class.
180   */
181  private static final long serialVersionUID = 3242469381380526294L;
182
183
184
185  // Indicates whether a request has been made to stop running.
186  private final AtomicBoolean stopRequested;
187
188  // The argument used to indicate whether to generate output in CSV format.
189  private BooleanArgument csvFormat;
190
191  // The argument used to indicate whether to suppress information about error
192  // result codes.
193  private BooleanArgument suppressErrors;
194
195  // The argument used to specify the collection interval.
196  private IntegerArgument collectionInterval;
197
198  // The argument used to specify the number of search and modify iterations on
199  // a connection before it is closed and re-established.
200  private IntegerArgument iterationsBeforeReconnect;
201
202  // The argument used to specify the number of intervals.
203  private IntegerArgument numIntervals;
204
205  // The argument used to specify the number of threads.
206  private IntegerArgument numThreads;
207
208  // The argument used to specify the seed to use for the random number
209  // generator.
210  private IntegerArgument randomSeed;
211
212  // The target rate of operations per second.
213  private IntegerArgument ratePerSecond;
214
215  // The argument used to specify a variable rate file.
216  private FileArgument sampleRateFile;
217
218  // The argument used to specify a variable rate file.
219  private FileArgument variableRateData;
220
221  // The argument used to specify the length of the values to generate.
222  private IntegerArgument valueLength;
223
224  // The number of warm-up intervals to perform.
225  private IntegerArgument warmUpIntervals;
226
227  // The argument used to specify the scope for the searches.
228  private ScopeArgument scopeArg;
229
230  // The argument used to specify the base DNs for the searches.
231  private StringArgument baseDN;
232
233  // The argument used to specify the set of characters to use when generating
234  // values.
235  private StringArgument characterSet;
236
237  // The argument used to specify the filters for the searches.
238  private StringArgument filter;
239
240  // The argument used to specify the attributes to modify.
241  private StringArgument modifyAttributes;
242
243  // The argument used to specify the proxied authorization identity.
244  private StringArgument proxyAs;
245
246  // The argument used to specify the attributes to return.
247  private StringArgument returnAttributes;
248
249  // The argument used to specify the timestamp format.
250  private StringArgument timestampFormat;
251
252  // The thread currently being used to run the searchrate tool.
253  private volatile Thread runningThread;
254
255  // A wakeable sleeper that will be used to sleep between reporting intervals.
256  private final WakeableSleeper sleeper;
257
258
259
260  /**
261   * Parse the provided command line arguments and make the appropriate set of
262   * changes.
263   *
264   * @param  args  The command line arguments provided to this program.
265   */
266  public static void main(final String[] args)
267  {
268    final ResultCode resultCode = main(args, System.out, System.err);
269    if (resultCode != ResultCode.SUCCESS)
270    {
271      System.exit(resultCode.intValue());
272    }
273  }
274
275
276
277  /**
278   * Parse the provided command line arguments and make the appropriate set of
279   * changes.
280   *
281   * @param  args       The command line arguments provided to this program.
282   * @param  outStream  The output stream to which standard out should be
283   *                    written.  It may be {@code null} if output should be
284   *                    suppressed.
285   * @param  errStream  The output stream to which standard error should be
286   *                    written.  It may be {@code null} if error messages
287   *                    should be suppressed.
288   *
289   * @return  A result code indicating whether the processing was successful.
290   */
291  public static ResultCode main(final String[] args,
292                                final OutputStream outStream,
293                                final OutputStream errStream)
294  {
295    final SearchAndModRate searchAndModRate =
296         new SearchAndModRate(outStream, errStream);
297    return searchAndModRate.runTool(args);
298  }
299
300
301
302  /**
303   * Creates a new instance of this tool.
304   *
305   * @param  outStream  The output stream to which standard out should be
306   *                    written.  It may be {@code null} if output should be
307   *                    suppressed.
308   * @param  errStream  The output stream to which standard error should be
309   *                    written.  It may be {@code null} if error messages
310   *                    should be suppressed.
311   */
312  public SearchAndModRate(final OutputStream outStream,
313                          final OutputStream errStream)
314  {
315    super(outStream, errStream);
316
317    stopRequested = new AtomicBoolean(false);
318    sleeper = new WakeableSleeper();
319  }
320
321
322
323  /**
324   * Retrieves the name for this tool.
325   *
326   * @return  The name for this tool.
327   */
328  @Override()
329  public String getToolName()
330  {
331    return "search-and-mod-rate";
332  }
333
334
335
336  /**
337   * Retrieves the description for this tool.
338   *
339   * @return  The description for this tool.
340   */
341  @Override()
342  public String getToolDescription()
343  {
344    return "Perform repeated searches against an " +
345           "LDAP directory server and modify each entry returned.";
346  }
347
348
349
350  /**
351   * Retrieves the version string for this tool.
352   *
353   * @return  The version string for this tool.
354   */
355  @Override()
356  public String getToolVersion()
357  {
358    return Version.NUMERIC_VERSION_STRING;
359  }
360
361
362
363  /**
364   * Indicates whether this tool should provide support for an interactive mode,
365   * in which the tool offers a mode in which the arguments can be provided in
366   * a text-driven menu rather than requiring them to be given on the command
367   * line.  If interactive mode is supported, it may be invoked using the
368   * "--interactive" argument.  Alternately, if interactive mode is supported
369   * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
370   * interactive mode may be invoked by simply launching the tool without any
371   * arguments.
372   *
373   * @return  {@code true} if this tool supports interactive mode, or
374   *          {@code false} if not.
375   */
376  @Override()
377  public boolean supportsInteractiveMode()
378  {
379    return true;
380  }
381
382
383
384  /**
385   * Indicates whether this tool defaults to launching in interactive mode if
386   * the tool is invoked without any command-line arguments.  This will only be
387   * used if {@link #supportsInteractiveMode()} returns {@code true}.
388   *
389   * @return  {@code true} if this tool defaults to using interactive mode if
390   *          launched without any command-line arguments, or {@code false} if
391   *          not.
392   */
393  @Override()
394  public boolean defaultsToInteractiveMode()
395  {
396    return true;
397  }
398
399
400
401  /**
402   * Indicates whether this tool should provide arguments for redirecting output
403   * to a file.  If this method returns {@code true}, then the tool will offer
404   * an "--outputFile" argument that will specify the path to a file to which
405   * all standard output and standard error content will be written, and it will
406   * also offer a "--teeToStandardOut" argument that can only be used if the
407   * "--outputFile" argument is present and will cause all output to be written
408   * to both the specified output file and to standard output.
409   *
410   * @return  {@code true} if this tool should provide arguments for redirecting
411   *          output to a file, or {@code false} if not.
412   */
413  @Override()
414  protected boolean supportsOutputFile()
415  {
416    return true;
417  }
418
419
420
421  /**
422   * Indicates whether this tool should default to interactively prompting for
423   * the bind password if a password is required but no argument was provided
424   * to indicate how to get the password.
425   *
426   * @return  {@code true} if this tool should default to interactively
427   *          prompting for the bind password, or {@code false} if not.
428   */
429  @Override()
430  protected boolean defaultToPromptForBindPassword()
431  {
432    return true;
433  }
434
435
436
437  /**
438   * Indicates whether this tool supports the use of a properties file for
439   * specifying default values for arguments that aren't specified on the
440   * command line.
441   *
442   * @return  {@code true} if this tool supports the use of a properties file
443   *          for specifying default values for arguments that aren't specified
444   *          on the command line, or {@code false} if not.
445   */
446  @Override()
447  public boolean supportsPropertiesFile()
448  {
449    return true;
450  }
451
452
453
454  /**
455   * Indicates whether the LDAP-specific arguments should include alternate
456   * versions of all long identifiers that consist of multiple words so that
457   * they are available in both camelCase and dash-separated versions.
458   *
459   * @return  {@code true} if this tool should provide multiple versions of
460   *          long identifiers for LDAP-specific arguments, or {@code false} if
461   *          not.
462   */
463  @Override()
464  protected boolean includeAlternateLongIdentifiers()
465  {
466    return true;
467  }
468
469
470
471  /**
472   * Adds the arguments used by this program that aren't already provided by the
473   * generic {@code LDAPCommandLineTool} framework.
474   *
475   * @param  parser  The argument parser to which the arguments should be added.
476   *
477   * @throws  ArgumentException  If a problem occurs while adding the arguments.
478   */
479  @Override()
480  public void addNonLDAPArguments(final ArgumentParser parser)
481         throws ArgumentException
482  {
483    String description = "The base DN to use for the searches.  It may be a " +
484         "simple DN or a value pattern to specify a range of DNs (e.g., " +
485         "\"uid=user.[1-1000],ou=People,dc=example,dc=com\").  See " +
486         ValuePattern.PUBLIC_JAVADOC_URL + " for complete details about the " +
487         "value pattern syntax.  This must be provided.";
488    baseDN = new StringArgument('b', "baseDN", true, 1, "{dn}", description);
489    baseDN.setArgumentGroupName("Search And Modification Arguments");
490    baseDN.addLongIdentifier("base-dn");
491    parser.addArgument(baseDN);
492
493
494    description = "The scope to use for the searches.  It should be 'base', " +
495                  "'one', 'sub', or 'subord'.  If this is not provided, then " +
496                  "a default scope of 'sub' will be used.";
497    scopeArg = new ScopeArgument('s', "scope", false, "{scope}", description,
498                                 SearchScope.SUB);
499    scopeArg.setArgumentGroupName("Search And Modification Arguments");
500    parser.addArgument(scopeArg);
501
502
503    description = "The filter to use for the searches.  It may be a simple " +
504                  "filter or a value pattern to specify a range of filters " +
505                  "(e.g., \"(uid=user.[1-1000])\").  See " +
506                  ValuePattern.PUBLIC_JAVADOC_URL + " for complete details " +
507                  "about the value pattern syntax.  This must be provided.";
508    filter = new StringArgument('f', "filter", true, 1, "{filter}",
509                                description);
510    filter.setArgumentGroupName("Search And Modification Arguments");
511    parser.addArgument(filter);
512
513
514    description = "The name of an attribute to include in entries returned " +
515                  "from the searches.  Multiple attributes may be requested " +
516                  "by providing this argument multiple times.  If no request " +
517                  "attributes are provided, then the entries returned will " +
518                  "include all user attributes.";
519    returnAttributes = new StringArgument('A', "attribute", false, 0, "{name}",
520                                          description);
521    returnAttributes.setArgumentGroupName("Search And Modification Arguments");
522    parser.addArgument(returnAttributes);
523
524
525    description = "The name of the attribute to modify.  Multiple attributes " +
526                  "may be specified by providing this argument multiple " +
527                  "times.  At least one attribute must be specified.";
528    modifyAttributes = new StringArgument('m', "modifyAttribute", true, 0,
529                                          "{name}", description);
530    modifyAttributes.setArgumentGroupName("Search And Modification Arguments");
531    modifyAttributes.addLongIdentifier("modify-attribute");
532    parser.addArgument(modifyAttributes);
533
534
535    description = "The length in bytes to use when generating values for the " +
536                  "modifications.  If this is not provided, then a default " +
537                  "length of ten bytes will be used.";
538    valueLength = new IntegerArgument('l', "valueLength", true, 1, "{num}",
539                                      description, 1, Integer.MAX_VALUE, 10);
540    valueLength.setArgumentGroupName("Search And Modification Arguments");
541    valueLength.addLongIdentifier("value-length");
542    parser.addArgument(valueLength);
543
544
545    description = "The set of characters to use to generate the values for " +
546                  "the modifications.  It should only include ASCII " +
547                  "characters.  If this is not provided, then a default set " +
548                  "of lowercase alphabetic characters will be used.";
549    characterSet = new StringArgument('C', "characterSet", true, 1, "{chars}",
550                                      description,
551                                      "abcdefghijklmnopqrstuvwxyz");
552    characterSet.setArgumentGroupName("Search And Modification Arguments");
553    characterSet.addLongIdentifier("character-set");
554    parser.addArgument(characterSet);
555
556
557    description = "The number of threads to use to perform the searches.  If " +
558                  "this is not provided, then a default of one thread will " +
559                  "be used.";
560    numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}",
561                                     description, 1, Integer.MAX_VALUE, 1);
562    numThreads.setArgumentGroupName("Rate Management Arguments");
563    numThreads.addLongIdentifier("num-threads");
564    parser.addArgument(numThreads);
565
566
567    description = "The length of time in seconds between output lines.  If " +
568                  "this is not provided, then a default interval of five " +
569                  "seconds will be used.";
570    collectionInterval = new IntegerArgument('i', "intervalDuration", true, 1,
571                                             "{num}", description, 1,
572                                             Integer.MAX_VALUE, 5);
573    collectionInterval.setArgumentGroupName("Rate Management Arguments");
574    collectionInterval.addLongIdentifier("interval-duration");
575    parser.addArgument(collectionInterval);
576
577
578    description = "The maximum number of intervals for which to run.  If " +
579                  "this is not provided, then the tool will run until it is " +
580                  "interrupted.";
581    numIntervals = new IntegerArgument('I', "numIntervals", true, 1, "{num}",
582                                       description, 1, Integer.MAX_VALUE,
583                                       Integer.MAX_VALUE);
584    numIntervals.setArgumentGroupName("Rate Management Arguments");
585    numIntervals.addLongIdentifier("num-intervals");
586    parser.addArgument(numIntervals);
587
588    description = "The number of search and modify iterations that should be " +
589                  "processed on a connection before that connection is " +
590                  "closed and replaced with a newly-established (and " +
591                  "authenticated, if appropriate) connection.  If this is " +
592                  "not provided, then connections will not be periodically " +
593                  "closed and re-established.";
594    iterationsBeforeReconnect = new IntegerArgument(null,
595         "iterationsBeforeReconnect", false, 1, "{num}", description, 0);
596    iterationsBeforeReconnect.setArgumentGroupName("Rate Management Arguments");
597    iterationsBeforeReconnect.addLongIdentifier("iterations-before-reconnect");
598    parser.addArgument(iterationsBeforeReconnect);
599
600    description = "The target number of searches to perform per second.  It " +
601                  "is still necessary to specify a sufficient number of " +
602                  "threads for achieving this rate.  If neither this option " +
603                  "nor --variableRateData is provided, then the tool will " +
604                  "run at the maximum rate for the specified number of " +
605                  "threads.";
606    ratePerSecond = new IntegerArgument('r', "ratePerSecond", false, 1,
607                                        "{searches-per-second}", description,
608                                        1, Integer.MAX_VALUE);
609    ratePerSecond.setArgumentGroupName("Rate Management Arguments");
610    ratePerSecond.addLongIdentifier("rate-per-second");
611    parser.addArgument(ratePerSecond);
612
613    final String variableRateDataArgName = "variableRateData";
614    final String generateSampleRateFileArgName = "generateSampleRateFile";
615    description = RateAdjustor.getVariableRateDataArgumentDescription(
616         generateSampleRateFileArgName);
617    variableRateData = new FileArgument(null, variableRateDataArgName, false, 1,
618                                        "{path}", description, true, true, true,
619                                        false);
620    variableRateData.setArgumentGroupName("Rate Management Arguments");
621    variableRateData.addLongIdentifier("variable-rate-data");
622    parser.addArgument(variableRateData);
623
624    description = RateAdjustor.getGenerateSampleVariableRateFileDescription(
625         variableRateDataArgName);
626    sampleRateFile = new FileArgument(null, generateSampleRateFileArgName,
627                                      false, 1, "{path}", description, false,
628                                      true, true, false);
629    sampleRateFile.setArgumentGroupName("Rate Management Arguments");
630    sampleRateFile.addLongIdentifier("generate-sample-rate-file");
631    sampleRateFile.setUsageArgument(true);
632    parser.addArgument(sampleRateFile);
633    parser.addExclusiveArgumentSet(variableRateData, sampleRateFile);
634
635    description = "The number of intervals to complete before beginning " +
636                  "overall statistics collection.  Specifying a nonzero " +
637                  "number of warm-up intervals gives the client and server " +
638                  "a chance to warm up without skewing performance results.";
639    warmUpIntervals = new IntegerArgument(null, "warmUpIntervals", true, 1,
640         "{num}", description, 0, Integer.MAX_VALUE, 0);
641    warmUpIntervals.setArgumentGroupName("Rate Management Arguments");
642    warmUpIntervals.addLongIdentifier("warm-up-intervals");
643    parser.addArgument(warmUpIntervals);
644
645    description = "Indicates the format to use for timestamps included in " +
646                  "the output.  A value of 'none' indicates that no " +
647                  "timestamps should be included.  A value of 'with-date' " +
648                  "indicates that both the date and the time should be " +
649                  "included.  A value of 'without-date' indicates that only " +
650                  "the time should be included.";
651    final LinkedHashSet<String> allowedFormats = new LinkedHashSet<String>(3);
652    allowedFormats.add("none");
653    allowedFormats.add("with-date");
654    allowedFormats.add("without-date");
655    timestampFormat = new StringArgument(null, "timestampFormat", true, 1,
656         "{format}", description, allowedFormats, "none");
657    timestampFormat.addLongIdentifier("timestamp-format");
658    parser.addArgument(timestampFormat);
659
660    description = "Indicates that the proxied authorization control (as " +
661                  "defined in RFC 4370) should be used to request that " +
662                  "operations be processed using an alternate authorization " +
663                  "identity.  This may be a simple authorization ID or it " +
664                  "may be a value pattern to specify a range of " +
665                  "identities.  See " + ValuePattern.PUBLIC_JAVADOC_URL +
666                  " for complete details about the value pattern syntax.";
667    proxyAs = new StringArgument('Y', "proxyAs", false, 1, "{authzID}",
668                                 description);
669    proxyAs.addLongIdentifier("proxy-as");
670    parser.addArgument(proxyAs);
671
672    description = "Indicates that information about the result codes for " +
673                  "failed operations should not be displayed.";
674    suppressErrors = new BooleanArgument(null,
675         "suppressErrorResultCodes", 1, description);
676    suppressErrors.addLongIdentifier("suppress-error-result-codes");
677    parser.addArgument(suppressErrors);
678
679    description = "Generate output in CSV format rather than a " +
680                  "display-friendly format";
681    csvFormat = new BooleanArgument('c', "csv", 1, description);
682    parser.addArgument(csvFormat);
683
684    description = "Specifies the seed to use for the random number generator.";
685    randomSeed = new IntegerArgument('R', "randomSeed", false, 1, "{value}",
686         description);
687    randomSeed.addLongIdentifier("random-seed");
688    parser.addArgument(randomSeed);
689  }
690
691
692
693  /**
694   * Indicates whether this tool supports creating connections to multiple
695   * servers.  If it is to support multiple servers, then the "--hostname" and
696   * "--port" arguments will be allowed to be provided multiple times, and
697   * will be required to be provided the same number of times.  The same type of
698   * communication security and bind credentials will be used for all servers.
699   *
700   * @return  {@code true} if this tool supports creating connections to
701   *          multiple servers, or {@code false} if not.
702   */
703  @Override()
704  protected boolean supportsMultipleServers()
705  {
706    return true;
707  }
708
709
710
711  /**
712   * Retrieves the connection options that should be used for connections
713   * created for use with this tool.
714   *
715   * @return  The connection options that should be used for connections created
716   *          for use with this tool.
717   */
718  @Override()
719  public LDAPConnectionOptions getConnectionOptions()
720  {
721    final LDAPConnectionOptions options = new LDAPConnectionOptions();
722    options.setUseSynchronousMode(true);
723    return options;
724  }
725
726
727
728  /**
729   * Performs the actual processing for this tool.  In this case, it gets a
730   * connection to the directory server and uses it to perform the requested
731   * searches.
732   *
733   * @return  The result code for the processing that was performed.
734   */
735  @Override()
736  public ResultCode doToolProcessing()
737  {
738    runningThread = Thread.currentThread();
739
740    try
741    {
742      return doToolProcessingInternal();
743    }
744    finally
745    {
746      runningThread = null;
747    }
748  }
749
750
751
752  /**
753   * Performs the actual processing for this tool.  In this case, it gets a
754   * connection to the directory server and uses it to perform the requested
755   * searches.
756   *
757   * @return  The result code for the processing that was performed.
758   */
759  private ResultCode doToolProcessingInternal()
760  {
761    // If the sample rate file argument was specified, then generate the sample
762    // variable rate data file and return.
763    if (sampleRateFile.isPresent())
764    {
765      try
766      {
767        RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue());
768        return ResultCode.SUCCESS;
769      }
770      catch (final Exception e)
771      {
772        debugException(e);
773        err("An error occurred while trying to write sample variable data " +
774             "rate file '", sampleRateFile.getValue().getAbsolutePath(),
775             "':  ", getExceptionMessage(e));
776        return ResultCode.LOCAL_ERROR;
777      }
778    }
779
780
781    // Determine the random seed to use.
782    final Long seed;
783    if (randomSeed.isPresent())
784    {
785      seed = Long.valueOf(randomSeed.getValue());
786    }
787    else
788    {
789      seed = null;
790    }
791
792    // Create value patterns for the base DN, filter, and proxied authorization
793    // DN.
794    final ValuePattern dnPattern;
795    try
796    {
797      dnPattern = new ValuePattern(baseDN.getValue(), seed);
798    }
799    catch (final ParseException pe)
800    {
801      debugException(pe);
802      err("Unable to parse the base DN value pattern:  ", pe.getMessage());
803      return ResultCode.PARAM_ERROR;
804    }
805
806    final ValuePattern filterPattern;
807    try
808    {
809      filterPattern = new ValuePattern(filter.getValue(), seed);
810    }
811    catch (final ParseException pe)
812    {
813      debugException(pe);
814      err("Unable to parse the filter pattern:  ", pe.getMessage());
815      return ResultCode.PARAM_ERROR;
816    }
817
818    final ValuePattern authzIDPattern;
819    if (proxyAs.isPresent())
820    {
821      try
822      {
823        authzIDPattern = new ValuePattern(proxyAs.getValue(), seed);
824      }
825      catch (final ParseException pe)
826      {
827        debugException(pe);
828        err("Unable to parse the proxied authorization pattern:  ",
829            pe.getMessage());
830        return ResultCode.PARAM_ERROR;
831      }
832    }
833    else
834    {
835      authzIDPattern = null;
836    }
837
838
839    // Get the attributes to return.
840    final String[] returnAttrs;
841    if (returnAttributes.isPresent())
842    {
843      final List<String> attrList = returnAttributes.getValues();
844      returnAttrs = new String[attrList.size()];
845      attrList.toArray(returnAttrs);
846    }
847    else
848    {
849      returnAttrs = NO_STRINGS;
850    }
851
852
853    // Get the names of the attributes to modify.
854    final String[] modAttrs = new String[modifyAttributes.getValues().size()];
855    modifyAttributes.getValues().toArray(modAttrs);
856
857
858    // Get the character set as a byte array.
859    final byte[] charSet = getBytes(characterSet.getValue());
860
861
862    // If the --ratePerSecond option was specified, then limit the rate
863    // accordingly.
864    FixedRateBarrier fixedRateBarrier = null;
865    if (ratePerSecond.isPresent() || variableRateData.isPresent())
866    {
867      // We might not have a rate per second if --variableRateData is specified.
868      // The rate typically doesn't matter except when we have warm-up
869      // intervals.  In this case, we'll run at the max rate.
870      final int intervalSeconds = collectionInterval.getValue();
871      final int ratePerInterval =
872           (ratePerSecond.getValue() == null)
873           ? Integer.MAX_VALUE
874           : ratePerSecond.getValue() * intervalSeconds;
875      fixedRateBarrier =
876           new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval);
877    }
878
879
880    // If --variableRateData was specified, then initialize a RateAdjustor.
881    RateAdjustor rateAdjustor = null;
882    if (variableRateData.isPresent())
883    {
884      try
885      {
886        rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier,
887             ratePerSecond.getValue(), variableRateData.getValue());
888      }
889      catch (final IOException e)
890      {
891        debugException(e);
892        err("Initializing the variable rates failed: " + e.getMessage());
893        return ResultCode.PARAM_ERROR;
894      }
895      catch (final IllegalArgumentException e)
896      {
897        debugException(e);
898        err("Initializing the variable rates failed: " + e.getMessage());
899        return ResultCode.PARAM_ERROR;
900      }
901    }
902
903
904    // Determine whether to include timestamps in the output and if so what
905    // format should be used for them.
906    final boolean includeTimestamp;
907    final String timeFormat;
908    if (timestampFormat.getValue().equalsIgnoreCase("with-date"))
909    {
910      includeTimestamp = true;
911      timeFormat       = "dd/MM/yyyy HH:mm:ss";
912    }
913    else if (timestampFormat.getValue().equalsIgnoreCase("without-date"))
914    {
915      includeTimestamp = true;
916      timeFormat       = "HH:mm:ss";
917    }
918    else
919    {
920      includeTimestamp = false;
921      timeFormat       = null;
922    }
923
924
925    // Determine whether any warm-up intervals should be run.
926    final long totalIntervals;
927    final boolean warmUp;
928    int remainingWarmUpIntervals = warmUpIntervals.getValue();
929    if (remainingWarmUpIntervals > 0)
930    {
931      warmUp = true;
932      totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals;
933    }
934    else
935    {
936      warmUp = true;
937      totalIntervals = 0L + numIntervals.getValue();
938    }
939
940
941    // Create the table that will be used to format the output.
942    final OutputFormat outputFormat;
943    if (csvFormat.isPresent())
944    {
945      outputFormat = OutputFormat.CSV;
946    }
947    else
948    {
949      outputFormat = OutputFormat.COLUMNS;
950    }
951
952    final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp,
953         timeFormat, outputFormat, " ",
954         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
955                  "Searches/Sec"),
956         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
957                  "Srch Dur ms"),
958         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
959                  "Mods/Sec"),
960         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
961                  "Mod Dur ms"),
962         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
963                  "Errors/Sec"),
964         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
965                  "Searches/Sec"),
966         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
967                  "Srch Dur ms"),
968         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
969                  "Mods/Sec"),
970         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
971                  "Mod Dur ms"));
972
973
974    // Create values to use for statistics collection.
975    final AtomicLong        searchCounter   = new AtomicLong(0L);
976    final AtomicLong        errorCounter    = new AtomicLong(0L);
977    final AtomicLong        modCounter      = new AtomicLong(0L);
978    final AtomicLong        modDurations    = new AtomicLong(0L);
979    final AtomicLong        searchDurations = new AtomicLong(0L);
980    final ResultCodeCounter rcCounter       = new ResultCodeCounter();
981
982
983    // Determine the length of each interval in milliseconds.
984    final long intervalMillis = 1000L * collectionInterval.getValue();
985
986
987    // Create the threads to use for the searches.
988    final Random random = new Random();
989    final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1);
990    final SearchAndModRateThread[] threads =
991         new SearchAndModRateThread[numThreads.getValue()];
992    for (int i=0; i < threads.length; i++)
993    {
994      final LDAPConnection connection;
995      try
996      {
997        connection = getConnection();
998      }
999      catch (final LDAPException le)
1000      {
1001        debugException(le);
1002        err("Unable to connect to the directory server:  ",
1003            getExceptionMessage(le));
1004        return le.getResultCode();
1005      }
1006
1007      threads[i] = new SearchAndModRateThread(this, i, connection, dnPattern,
1008           scopeArg.getValue(), filterPattern, returnAttrs, modAttrs,
1009           valueLength.getValue(), charSet, authzIDPattern,
1010           iterationsBeforeReconnect.getValue(), random.nextLong(), barrier,
1011           searchCounter, modCounter, searchDurations, modDurations,
1012           errorCounter, rcCounter, fixedRateBarrier);
1013      threads[i].start();
1014    }
1015
1016
1017    // Display the table header.
1018    for (final String headerLine : formatter.getHeaderLines(true))
1019    {
1020      out(headerLine);
1021    }
1022
1023
1024    // Start the RateAdjustor before the threads so that the initial value is
1025    // in place before any load is generated unless we're doing a warm-up in
1026    // which case, we'll start it after the warm-up is complete.
1027    if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0))
1028    {
1029      rateAdjustor.start();
1030    }
1031
1032
1033    // Indicate that the threads can start running.
1034    try
1035    {
1036      barrier.await();
1037    }
1038    catch (final Exception e)
1039    {
1040      debugException(e);
1041    }
1042
1043    long overallStartTime = System.nanoTime();
1044    long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis;
1045
1046
1047    boolean setOverallStartTime = false;
1048    long    lastSearchDuration  = 0L;
1049    long    lastModDuration     = 0L;
1050    long    lastNumErrors       = 0L;
1051    long    lastNumSearches     = 0L;
1052    long    lastNumMods          = 0L;
1053    long    lastEndTime         = System.nanoTime();
1054    for (long i=0; i < totalIntervals; i++)
1055    {
1056      if (rateAdjustor != null)
1057      {
1058        if (! rateAdjustor.isAlive())
1059        {
1060          out("All of the rates in " + variableRateData.getValue().getName() +
1061              " have been completed.");
1062          break;
1063        }
1064      }
1065
1066      final long startTimeMillis = System.currentTimeMillis();
1067      final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis;
1068      nextIntervalStartTime += intervalMillis;
1069      if (sleepTimeMillis > 0)
1070      {
1071        sleeper.sleep(sleepTimeMillis);
1072      }
1073
1074      if (stopRequested.get())
1075      {
1076        break;
1077      }
1078
1079      final long endTime          = System.nanoTime();
1080      final long intervalDuration = endTime - lastEndTime;
1081
1082      final long numSearches;
1083      final long numMods;
1084      final long numErrors;
1085      final long totalSearchDuration;
1086      final long totalModDuration;
1087      if (warmUp && (remainingWarmUpIntervals > 0))
1088      {
1089        numSearches         = searchCounter.getAndSet(0L);
1090        numMods             = modCounter.getAndSet(0L);
1091        numErrors           = errorCounter.getAndSet(0L);
1092        totalSearchDuration = searchDurations.getAndSet(0L);
1093        totalModDuration    = modDurations.getAndSet(0L);
1094      }
1095      else
1096      {
1097        numSearches         = searchCounter.get();
1098        numMods             = modCounter.get();
1099        numErrors           = errorCounter.get();
1100        totalSearchDuration = searchDurations.get();
1101        totalModDuration    = modDurations.get();
1102      }
1103
1104      final long recentNumSearches = numSearches - lastNumSearches;
1105      final long recentNumMods = numMods - lastNumMods;
1106      final long recentNumErrors = numErrors - lastNumErrors;
1107      final long recentSearchDuration =
1108           totalSearchDuration - lastSearchDuration;
1109      final long recentModDuration = totalModDuration - lastModDuration;
1110
1111      final double numSeconds = intervalDuration / 1000000000.0d;
1112      final double recentSearchRate = recentNumSearches / numSeconds;
1113      final double recentModRate = recentNumMods / numSeconds;
1114      final double recentErrorRate  = recentNumErrors / numSeconds;
1115
1116      final double recentAvgSearchDuration;
1117      if (recentNumSearches > 0L)
1118      {
1119        recentAvgSearchDuration =
1120             1.0d * recentSearchDuration / recentNumSearches / 1000000;
1121      }
1122      else
1123      {
1124        recentAvgSearchDuration = 0.0d;
1125      }
1126
1127      final double recentAvgModDuration;
1128      if (recentNumMods > 0L)
1129      {
1130        recentAvgModDuration =
1131             1.0d * recentModDuration / recentNumMods / 1000000;
1132      }
1133      else
1134      {
1135        recentAvgModDuration = 0.0d;
1136      }
1137
1138      if (warmUp && (remainingWarmUpIntervals > 0))
1139      {
1140        out(formatter.formatRow(recentSearchRate, recentAvgSearchDuration,
1141             recentModRate, recentAvgModDuration, recentErrorRate, "warming up",
1142             "warming up", "warming up", "warming up"));
1143
1144        remainingWarmUpIntervals--;
1145        if (remainingWarmUpIntervals == 0)
1146        {
1147          out("Warm-up completed.  Beginning overall statistics collection.");
1148          setOverallStartTime = true;
1149          if (rateAdjustor != null)
1150          {
1151            rateAdjustor.start();
1152          }
1153        }
1154      }
1155      else
1156      {
1157        if (setOverallStartTime)
1158        {
1159          overallStartTime    = lastEndTime;
1160          setOverallStartTime = false;
1161        }
1162
1163        final double numOverallSeconds =
1164             (endTime - overallStartTime) / 1000000000.0d;
1165        final double overallSearchRate = numSearches / numOverallSeconds;
1166        final double overallModRate = numMods / numOverallSeconds;
1167
1168        final double overallAvgSearchDuration;
1169        if (numSearches > 0L)
1170        {
1171          overallAvgSearchDuration =
1172               1.0d * totalSearchDuration / numSearches / 1000000;
1173        }
1174        else
1175        {
1176          overallAvgSearchDuration = 0.0d;
1177        }
1178
1179        final double overallAvgModDuration;
1180        if (numMods > 0L)
1181        {
1182          overallAvgModDuration =
1183               1.0d * totalModDuration / numMods / 1000000;
1184        }
1185        else
1186        {
1187          overallAvgModDuration = 0.0d;
1188        }
1189
1190        out(formatter.formatRow(recentSearchRate, recentAvgSearchDuration,
1191             recentModRate, recentAvgModDuration, recentErrorRate,
1192             overallSearchRate, overallAvgSearchDuration, overallModRate,
1193             overallAvgModDuration));
1194
1195        lastNumSearches    = numSearches;
1196        lastNumMods        = numMods;
1197        lastNumErrors      = numErrors;
1198        lastSearchDuration = totalSearchDuration;
1199        lastModDuration    = totalModDuration;
1200      }
1201
1202      final List<ObjectPair<ResultCode,Long>> rcCounts =
1203           rcCounter.getCounts(true);
1204      if ((! suppressErrors.isPresent()) && (! rcCounts.isEmpty()))
1205      {
1206        err("\tError Results:");
1207        for (final ObjectPair<ResultCode,Long> p : rcCounts)
1208        {
1209          err("\t", p.getFirst().getName(), ":  ", p.getSecond());
1210        }
1211      }
1212
1213      lastEndTime = endTime;
1214    }
1215
1216
1217    // Shut down the RateAdjustor if we have one.
1218    if (rateAdjustor != null)
1219    {
1220      rateAdjustor.shutDown();
1221    }
1222
1223    // Stop all of the threads.
1224    ResultCode resultCode = ResultCode.SUCCESS;
1225    for (final SearchAndModRateThread t : threads)
1226    {
1227      final ResultCode r = t.stopRunning();
1228      if (resultCode == ResultCode.SUCCESS)
1229      {
1230        resultCode = r;
1231      }
1232    }
1233
1234    return resultCode;
1235  }
1236
1237
1238
1239  /**
1240   * Requests that this tool stop running.  This method will attempt to wait
1241   * for all threads to complete before returning control to the caller.
1242   */
1243  public void stopRunning()
1244  {
1245    stopRequested.set(true);
1246    sleeper.wakeup();
1247
1248    final Thread t = runningThread;
1249    if (t != null)
1250    {
1251      try
1252      {
1253        t.join();
1254      }
1255      catch (final Exception e)
1256      {
1257        debugException(e);
1258      }
1259    }
1260  }
1261
1262
1263
1264  /**
1265   * {@inheritDoc}
1266   */
1267  @Override()
1268  public LinkedHashMap<String[],String> getExampleUsages()
1269  {
1270    final LinkedHashMap<String[],String> examples =
1271         new LinkedHashMap<String[],String>(2);
1272
1273    String[] args =
1274    {
1275      "--hostname", "server.example.com",
1276      "--port", "389",
1277      "--bindDN", "uid=admin,dc=example,dc=com",
1278      "--bindPassword", "password",
1279      "--baseDN", "dc=example,dc=com",
1280      "--scope", "sub",
1281      "--filter", "(uid=user.[1-1000000])",
1282      "--attribute", "givenName",
1283      "--attribute", "sn",
1284      "--attribute", "mail",
1285      "--modifyAttribute", "description",
1286      "--valueLength", "10",
1287      "--characterSet", "abcdefghijklmnopqrstuvwxyz0123456789",
1288      "--numThreads", "10"
1289    };
1290    String description =
1291         "Test search and modify performance by searching randomly across a " +
1292         "set of one million users located below 'dc=example,dc=com' with " +
1293         "ten concurrent threads.  The entries returned to the client will " +
1294         "include the givenName, sn, and mail attributes, and the " +
1295         "description attribute of each entry returned will be replaced " +
1296         "with a string of ten randomly-selected alphanumeric characters.";
1297    examples.put(args, description);
1298
1299    args = new String[]
1300    {
1301      "--generateSampleRateFile", "variable-rate-data.txt"
1302    };
1303    description =
1304         "Generate a sample variable rate definition file that may be used " +
1305         "in conjunction with the --variableRateData argument.  The sample " +
1306         "file will include comments that describe the format for data to be " +
1307         "included in this file.";
1308    examples.put(args, description);
1309
1310    return examples;
1311  }
1312}