001/*
002 * Copyright 2008-2016 UnboundID Corp.
003 * All Rights Reserved.
004 */
005/*
006 * Copyright (C) 2008-2016 UnboundID Corp.
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021package com.unboundid.ldap.sdk.examples;
022
023
024
025import java.io.IOException;
026import java.io.OutputStream;
027import java.io.Serializable;
028import java.text.ParseException;
029import java.util.LinkedHashMap;
030import java.util.LinkedHashSet;
031import java.util.List;
032import java.util.concurrent.CyclicBarrier;
033import java.util.concurrent.Semaphore;
034import java.util.concurrent.atomic.AtomicBoolean;
035import java.util.concurrent.atomic.AtomicLong;
036
037import com.unboundid.ldap.sdk.LDAPConnection;
038import com.unboundid.ldap.sdk.LDAPConnectionOptions;
039import com.unboundid.ldap.sdk.LDAPException;
040import com.unboundid.ldap.sdk.ResultCode;
041import com.unboundid.ldap.sdk.SearchScope;
042import com.unboundid.ldap.sdk.Version;
043import com.unboundid.util.ColumnFormatter;
044import com.unboundid.util.FixedRateBarrier;
045import com.unboundid.util.FormattableColumn;
046import com.unboundid.util.HorizontalAlignment;
047import com.unboundid.util.LDAPCommandLineTool;
048import com.unboundid.util.ObjectPair;
049import com.unboundid.util.OutputFormat;
050import com.unboundid.util.RateAdjustor;
051import com.unboundid.util.ResultCodeCounter;
052import com.unboundid.util.ThreadSafety;
053import com.unboundid.util.ThreadSafetyLevel;
054import com.unboundid.util.WakeableSleeper;
055import com.unboundid.util.ValuePattern;
056import com.unboundid.util.args.ArgumentException;
057import com.unboundid.util.args.ArgumentParser;
058import com.unboundid.util.args.BooleanArgument;
059import com.unboundid.util.args.FileArgument;
060import com.unboundid.util.args.IntegerArgument;
061import com.unboundid.util.args.ScopeArgument;
062import com.unboundid.util.args.StringArgument;
063
064import static com.unboundid.util.Debug.*;
065import static com.unboundid.util.StaticUtils.*;
066
067
068
069/**
070 * This class provides a tool that can be used to search an LDAP directory
071 * server repeatedly using multiple threads.  It can help provide an estimate of
072 * the search performance that a directory server is able to achieve.  Either or
073 * both of the base DN and the search filter may be a value pattern as
074 * described in the {@link ValuePattern} class.  This makes it possible to
075 * search over a range of entries rather than repeatedly performing searches
076 * with the same base DN and filter.
077 * <BR><BR>
078 * Some of the APIs demonstrated by this example include:
079 * <UL>
080 *   <LI>Argument Parsing (from the {@code com.unboundid.util.args}
081 *       package)</LI>
082 *   <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util}
083 *       package)</LI>
084 *   <LI>LDAP Communication (from the {@code com.unboundid.ldap.sdk}
085 *       package)</LI>
086 *   <LI>Value Patterns (from the {@code com.unboundid.util} package)</LI>
087 * </UL>
088 * <BR><BR>
089 * All of the necessary information is provided using command line arguments.
090 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
091 * class, as well as the following additional arguments:
092 * <UL>
093 *   <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use
094 *       for the searches.  This must be provided.  It may be a simple DN, or it
095 *       may be a value pattern to express a range of base DNs.</LI>
096 *   <LI>"-s {scope}" or "--scope {scope}" -- specifies the scope to use for the
097 *       search.  The scope value should be one of "base", "one", "sub", or
098 *       "subord".  If this isn't specified, then a scope of "sub" will be
099 *       used.</LI>
100 *   <LI>"-f {filter}" or "--filter {filter}" -- specifies the filter to use for
101 *       the searches.  This must be provided.  It may be a simple filter, or it
102 *       may be a value pattern to express a range of filters.</LI>
103 *   <LI>"-A {name}" or "--attribute {name}" -- specifies the name of an
104 *       attribute that should be included in entries returned from the server.
105 *       If this is not provided, then all user attributes will be requested.
106 *       This may include special tokens that the server may interpret, like
107 *       "1.1" to indicate that no attributes should be returned, "*", for all
108 *       user attributes, or "+" for all operational attributes.  Multiple
109 *       attributes may be requested with multiple instances of this
110 *       argument.</LI>
111 *   <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of
112 *       concurrent threads to use when performing the searches.  If this is not
113 *       provided, then a default of one thread will be used.</LI>
114 *   <LI>"-i {sec}" or "--intervalDuration {sec}" -- specifies the length of
115 *       time in seconds between lines out output.  If this is not provided,
116 *       then a default interval duration of five seconds will be used.</LI>
117 *   <LI>"-I {num}" or "--numIntervals {num}" -- specifies the maximum number of
118 *       intervals for which to run.  If this is not provided, then it will
119 *       run forever.</LI>
120 *   <LI>"--iterationsBeforeReconnect {num}" -- specifies the number of search
121 *       iterations that should be performed on a connection before that
122 *       connection is closed and replaced with a newly-established (and
123 *       authenticated, if appropriate) connection.</LI>
124 *   <LI>"-r {searches-per-second}" or "--ratePerSecond {searches-per-second}"
125 *       -- specifies the target number of searches to perform per second.  It
126 *       is still necessary to specify a sufficient number of threads for
127 *       achieving this rate.  If this option is not provided, then the tool
128 *       will run at the maximum rate for the specified number of threads.</LI>
129 *   <LI>"--variableRateData {path}" -- specifies the path to a file containing
130 *       information needed to allow the tool to vary the target rate over time.
131 *       If this option is not provided, then the tool will either use a fixed
132 *       target rate as specified by the "--ratePerSecond" argument, or it will
133 *       run at the maximum rate.</LI>
134 *   <LI>"--generateSampleRateFile {path}" -- specifies the path to a file to
135 *       which sample data will be written illustrating and describing the
136 *       format of the file expected to be used in conjunction with the
137 *       "--variableRateData" argument.</LI>
138 *   <LI>"--warmUpIntervals {num}" -- specifies the number of intervals to
139 *       complete before beginning overall statistics collection.</LI>
140 *   <LI>"--timestampFormat {format}" -- specifies the format to use for
141 *       timestamps included before each output line.  The format may be one of
142 *       "none" (for no timestamps), "with-date" (to include both the date and
143 *       the time), or "without-date" (to include only time time).</LI>
144 *   <LI>"-Y {authzID}" or "--proxyAs {authzID}" -- Use the proxied
145 *       authorization v2 control to request that the operation be processed
146 *       using an alternate authorization identity.  In this case, the bind DN
147 *       should be that of a user that has permission to use this control.  The
148 *       authorization identity may be a value pattern.</LI>
149 *   <LI>"-a" or "--asynchronous" -- Indicates that searches should be performed
150 *       in asynchronous mode, in which the client will not wait for a response
151 *       to a previous request before sending the next request.  Either the
152 *       "--ratePerSecond" or "--maxOutstandingRequests" arguments must be
153 *       provided to limit the number of outstanding requests.</LI>
154 *   <LI>"-O {num}" or "--maxOutstandingRequests {num}" -- Specifies the maximum
155 *       number of outstanding requests that will be allowed in asynchronous
156 *       mode.</LI>
157 *   <LI>"--suppressErrorResultCodes" -- Indicates that information about the
158 *       result codes for failed operations should not be displayed.</LI>
159 *   <LI>"-c" or "--csv" -- Generate output in CSV format rather than a
160 *       display-friendly format.</LI>
161 * </UL>
162 */
163@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
164public final class SearchRate
165       extends LDAPCommandLineTool
166       implements Serializable
167{
168  /**
169   * The serial version UID for this serializable class.
170   */
171  private static final long serialVersionUID = 3345838530404592182L;
172
173
174
175  // Indicates whether a request has been made to stop running.
176  private final AtomicBoolean stopRequested;
177
178  // The argument used to indicate whether to operate in asynchronous mode.
179  private BooleanArgument asynchronousMode;
180
181  // The argument used to indicate whether to generate output in CSV format.
182  private BooleanArgument csvFormat;
183
184  // The argument used to indicate whether to suppress information about error
185  // result codes.
186  private BooleanArgument suppressErrors;
187
188  // The argument used to specify the collection interval.
189  private IntegerArgument collectionInterval;
190
191  // The argument used to specify the number of search iterations on a
192  // connection before it is closed and re-established.
193  private IntegerArgument iterationsBeforeReconnect;
194
195  // The argument used to specify the maximum number of outstanding asynchronous
196  // requests.
197  private IntegerArgument maxOutstandingRequests;
198
199  // The argument used to specify the number of intervals.
200  private IntegerArgument numIntervals;
201
202  // The argument used to specify the number of threads.
203  private IntegerArgument numThreads;
204
205  // The argument used to specify the seed to use for the random number
206  // generator.
207  private IntegerArgument randomSeed;
208
209  // The target rate of searches per second.
210  private IntegerArgument ratePerSecond;
211
212  // The argument used to specify a variable rate file.
213  private FileArgument sampleRateFile;
214
215  // The argument used to specify a variable rate file.
216  private FileArgument variableRateData;
217
218  // The number of warm-up intervals to perform.
219  private IntegerArgument warmUpIntervals;
220
221  // The argument used to specify the scope for the searches.
222  private ScopeArgument scopeArg;
223
224  // The argument used to specify the attributes to return.
225  private StringArgument attributes;
226
227  // The argument used to specify the base DNs for the searches.
228  private StringArgument baseDN;
229
230  // The argument used to specify the filters for the searches.
231  private StringArgument filter;
232
233  // The argument used to specify the proxied authorization identity.
234  private StringArgument proxyAs;
235
236  // The argument used to specify the timestamp format.
237  private StringArgument timestampFormat;
238
239  // The thread currently being used to run the searchrate tool.
240  private volatile Thread runningThread;
241
242  // A wakeable sleeper that will be used to sleep between reporting intervals.
243  private final WakeableSleeper sleeper;
244
245
246
247  /**
248   * Parse the provided command line arguments and make the appropriate set of
249   * changes.
250   *
251   * @param  args  The command line arguments provided to this program.
252   */
253  public static void main(final String[] args)
254  {
255    final ResultCode resultCode = main(args, System.out, System.err);
256    if (resultCode != ResultCode.SUCCESS)
257    {
258      System.exit(resultCode.intValue());
259    }
260  }
261
262
263
264  /**
265   * Parse the provided command line arguments and make the appropriate set of
266   * changes.
267   *
268   * @param  args       The command line arguments provided to this program.
269   * @param  outStream  The output stream to which standard out should be
270   *                    written.  It may be {@code null} if output should be
271   *                    suppressed.
272   * @param  errStream  The output stream to which standard error should be
273   *                    written.  It may be {@code null} if error messages
274   *                    should be suppressed.
275   *
276   * @return  A result code indicating whether the processing was successful.
277   */
278  public static ResultCode main(final String[] args,
279                                final OutputStream outStream,
280                                final OutputStream errStream)
281  {
282    final SearchRate searchRate = new SearchRate(outStream, errStream);
283    return searchRate.runTool(args);
284  }
285
286
287
288  /**
289   * Creates a new instance of this tool.
290   *
291   * @param  outStream  The output stream to which standard out should be
292   *                    written.  It may be {@code null} if output should be
293   *                    suppressed.
294   * @param  errStream  The output stream to which standard error should be
295   *                    written.  It may be {@code null} if error messages
296   *                    should be suppressed.
297   */
298  public SearchRate(final OutputStream outStream, final OutputStream errStream)
299  {
300    super(outStream, errStream);
301
302    stopRequested = new AtomicBoolean(false);
303    sleeper = new WakeableSleeper();
304  }
305
306
307
308  /**
309   * Retrieves the name for this tool.
310   *
311   * @return  The name for this tool.
312   */
313  @Override()
314  public String getToolName()
315  {
316    return "searchrate";
317  }
318
319
320
321  /**
322   * Retrieves the description for this tool.
323   *
324   * @return  The description for this tool.
325   */
326  @Override()
327  public String getToolDescription()
328  {
329    return "Perform repeated searches against an " +
330           "LDAP directory server.";
331  }
332
333
334
335  /**
336   * Retrieves the version string for this tool.
337   *
338   * @return  The version string for this tool.
339   */
340  @Override()
341  public String getToolVersion()
342  {
343    return Version.NUMERIC_VERSION_STRING;
344  }
345
346
347
348  /**
349   * Indicates whether this tool should provide support for an interactive mode,
350   * in which the tool offers a mode in which the arguments can be provided in
351   * a text-driven menu rather than requiring them to be given on the command
352   * line.  If interactive mode is supported, it may be invoked using the
353   * "--interactive" argument.  Alternately, if interactive mode is supported
354   * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
355   * interactive mode may be invoked by simply launching the tool without any
356   * arguments.
357   *
358   * @return  {@code true} if this tool supports interactive mode, or
359   *          {@code false} if not.
360   */
361  @Override()
362  public boolean supportsInteractiveMode()
363  {
364    return true;
365  }
366
367
368
369  /**
370   * Indicates whether this tool defaults to launching in interactive mode if
371   * the tool is invoked without any command-line arguments.  This will only be
372   * used if {@link #supportsInteractiveMode()} returns {@code true}.
373   *
374   * @return  {@code true} if this tool defaults to using interactive mode if
375   *          launched without any command-line arguments, or {@code false} if
376   *          not.
377   */
378  @Override()
379  public boolean defaultsToInteractiveMode()
380  {
381    return true;
382  }
383
384
385
386  /**
387   * Indicates whether this tool should provide arguments for redirecting output
388   * to a file.  If this method returns {@code true}, then the tool will offer
389   * an "--outputFile" argument that will specify the path to a file to which
390   * all standard output and standard error content will be written, and it will
391   * also offer a "--teeToStandardOut" argument that can only be used if the
392   * "--outputFile" argument is present and will cause all output to be written
393   * to both the specified output file and to standard output.
394   *
395   * @return  {@code true} if this tool should provide arguments for redirecting
396   *          output to a file, or {@code false} if not.
397   */
398  @Override()
399  protected boolean supportsOutputFile()
400  {
401    return true;
402  }
403
404
405
406  /**
407   * Indicates whether this tool should default to interactively prompting for
408   * the bind password if a password is required but no argument was provided
409   * to indicate how to get the password.
410   *
411   * @return  {@code true} if this tool should default to interactively
412   *          prompting for the bind password, or {@code false} if not.
413   */
414  @Override()
415  protected boolean defaultToPromptForBindPassword()
416  {
417    return true;
418  }
419
420
421
422  /**
423   * Indicates whether this tool supports the use of a properties file for
424   * specifying default values for arguments that aren't specified on the
425   * command line.
426   *
427   * @return  {@code true} if this tool supports the use of a properties file
428   *          for specifying default values for arguments that aren't specified
429   *          on the command line, or {@code false} if not.
430   */
431  @Override()
432  public boolean supportsPropertiesFile()
433  {
434    return true;
435  }
436
437
438
439  /**
440   * Indicates whether the LDAP-specific arguments should include alternate
441   * versions of all long identifiers that consist of multiple words so that
442   * they are available in both camelCase and dash-separated versions.
443   *
444   * @return  {@code true} if this tool should provide multiple versions of
445   *          long identifiers for LDAP-specific arguments, or {@code false} if
446   *          not.
447   */
448  @Override()
449  protected boolean includeAlternateLongIdentifiers()
450  {
451    return true;
452  }
453
454
455
456  /**
457   * Adds the arguments used by this program that aren't already provided by the
458   * generic {@code LDAPCommandLineTool} framework.
459   *
460   * @param  parser  The argument parser to which the arguments should be added.
461   *
462   * @throws  ArgumentException  If a problem occurs while adding the arguments.
463   */
464  @Override()
465  public void addNonLDAPArguments(final ArgumentParser parser)
466         throws ArgumentException
467  {
468    String description = "The base DN to use for the searches.  It may be a " +
469         "simple DN or a value pattern to specify a range of DNs (e.g., " +
470         "\"uid=user.[1-1000],ou=People,dc=example,dc=com\").  See " +
471         ValuePattern.PUBLIC_JAVADOC_URL + " for complete details about the " +
472         "value pattern syntax.  This must be provided.";
473    baseDN = new StringArgument('b', "baseDN", true, 1, "{dn}", description);
474    baseDN.setArgumentGroupName("Search Arguments");
475    baseDN.addLongIdentifier("base-dn");
476    parser.addArgument(baseDN);
477
478
479    description = "The scope to use for the searches.  It should be 'base', " +
480                  "'one', 'sub', or 'subord'.  If this is not provided, then " +
481                  "a default scope of 'sub' will be used.";
482    scopeArg = new ScopeArgument('s', "scope", false, "{scope}", description,
483                                 SearchScope.SUB);
484    scopeArg.setArgumentGroupName("Search Arguments");
485    parser.addArgument(scopeArg);
486
487
488    description = "The filter to use for the searches.  It may be a simple " +
489                  "filter or a value pattern to specify a range of filters " +
490                  "(e.g., \"(uid=user.[1-1000])\").  See " +
491                  ValuePattern.PUBLIC_JAVADOC_URL + " for complete details " +
492                  "about the value pattern syntax.  This must be provided.";
493    filter = new StringArgument('f', "filter", true, 1, "{filter}",
494                                description);
495    filter.setArgumentGroupName("Search Arguments");
496    parser.addArgument(filter);
497
498
499    description = "The name of an attribute to include in entries returned " +
500                  "from the searches.  Multiple attributes may be requested " +
501                  "by providing this argument multiple times.  If no request " +
502                  "attributes are provided, then the entries returned will " +
503                  "include all user attributes.";
504    attributes = new StringArgument('A', "attribute", false, 0, "{name}",
505                                    description);
506    attributes.setArgumentGroupName("Search Arguments");
507    parser.addArgument(attributes);
508
509
510    description = "The number of threads to use to perform the searches.  If " +
511                  "this is not provided, then a default of one thread will " +
512                  "be used.";
513    numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}",
514                                     description, 1, Integer.MAX_VALUE, 1);
515    numThreads.setArgumentGroupName("Rate Management Arguments");
516    numThreads.addLongIdentifier("num-threads");
517    parser.addArgument(numThreads);
518
519
520    description = "The length of time in seconds between output lines.  If " +
521                  "this is not provided, then a default interval of five " +
522                  "seconds will be used.";
523    collectionInterval = new IntegerArgument('i', "intervalDuration", true, 1,
524                                             "{num}", description, 1,
525                                             Integer.MAX_VALUE, 5);
526    collectionInterval.setArgumentGroupName("Rate Management Arguments");
527    collectionInterval.addLongIdentifier("interval-duration");
528    parser.addArgument(collectionInterval);
529
530
531    description = "The maximum number of intervals for which to run.  If " +
532                  "this is not provided, then the tool will run until it is " +
533                  "interrupted.";
534    numIntervals = new IntegerArgument('I', "numIntervals", true, 1, "{num}",
535                                       description, 1, Integer.MAX_VALUE,
536                                       Integer.MAX_VALUE);
537    numIntervals.setArgumentGroupName("Rate Management Arguments");
538    numIntervals.addLongIdentifier("num-intervals");
539    parser.addArgument(numIntervals);
540
541    description = "The number of search iterations that should be processed " +
542                  "on a connection before that connection is closed and " +
543                  "replaced with a newly-established (and authenticated, if " +
544                  "appropriate) connection.  If this is not provided, then " +
545                  "connections will not be periodically closed and " +
546                  "re-established.";
547    iterationsBeforeReconnect = new IntegerArgument(null,
548         "iterationsBeforeReconnect", false, 1, "{num}", description, 0);
549    iterationsBeforeReconnect.setArgumentGroupName("Rate Management Arguments");
550    iterationsBeforeReconnect.addLongIdentifier("iterations-before-reconnect");
551    parser.addArgument(iterationsBeforeReconnect);
552
553    description = "The target number of searches to perform per second.  It " +
554                  "is still necessary to specify a sufficient number of " +
555                  "threads for achieving this rate.  If neither this option " +
556                  "nor --variableRateData is provided, then the tool will " +
557                  "run at the maximum rate for the specified number of " +
558                  "threads.";
559    ratePerSecond = new IntegerArgument('r', "ratePerSecond", false, 1,
560                                        "{searches-per-second}", description,
561                                        1, Integer.MAX_VALUE);
562    ratePerSecond.setArgumentGroupName("Rate Management Arguments");
563    ratePerSecond.addLongIdentifier("rate-per-second");
564    parser.addArgument(ratePerSecond);
565
566    final String variableRateDataArgName = "variableRateData";
567    final String generateSampleRateFileArgName = "generateSampleRateFile";
568    description = RateAdjustor.getVariableRateDataArgumentDescription(
569         generateSampleRateFileArgName);
570    variableRateData = new FileArgument(null, variableRateDataArgName, false, 1,
571                                        "{path}", description, true, true, true,
572                                        false);
573    variableRateData.setArgumentGroupName("Rate Management Arguments");
574    variableRateData.addLongIdentifier("variable-rate-data");
575    parser.addArgument(variableRateData);
576
577    description = RateAdjustor.getGenerateSampleVariableRateFileDescription(
578         variableRateDataArgName);
579    sampleRateFile = new FileArgument(null, generateSampleRateFileArgName,
580                                      false, 1, "{path}", description, false,
581                                      true, true, false);
582    sampleRateFile.setArgumentGroupName("Rate Management Arguments");
583    sampleRateFile.addLongIdentifier("generate-sample-rate-file");
584    sampleRateFile.setUsageArgument(true);
585    parser.addArgument(sampleRateFile);
586    parser.addExclusiveArgumentSet(variableRateData, sampleRateFile);
587
588    description = "The number of intervals to complete before beginning " +
589                  "overall statistics collection.  Specifying a nonzero " +
590                  "number of warm-up intervals gives the client and server " +
591                  "a chance to warm up without skewing performance results.";
592    warmUpIntervals = new IntegerArgument(null, "warmUpIntervals", true, 1,
593         "{num}", description, 0, Integer.MAX_VALUE, 0);
594    warmUpIntervals.setArgumentGroupName("Rate Management Arguments");
595    warmUpIntervals.addLongIdentifier("warm-up-intervals");
596    parser.addArgument(warmUpIntervals);
597
598    description = "Indicates the format to use for timestamps included in " +
599                  "the output.  A value of 'none' indicates that no " +
600                  "timestamps should be included.  A value of 'with-date' " +
601                  "indicates that both the date and the time should be " +
602                  "included.  A value of 'without-date' indicates that only " +
603                  "the time should be included.";
604    final LinkedHashSet<String> allowedFormats = new LinkedHashSet<String>(3);
605    allowedFormats.add("none");
606    allowedFormats.add("with-date");
607    allowedFormats.add("without-date");
608    timestampFormat = new StringArgument(null, "timestampFormat", true, 1,
609         "{format}", description, allowedFormats, "none");
610    timestampFormat.addLongIdentifier("timestamp-format");
611    parser.addArgument(timestampFormat);
612
613    description = "Indicates that the proxied authorization control (as " +
614                  "defined in RFC 4370) should be used to request that " +
615                  "operations be processed using an alternate authorization " +
616                  "identity.  This may be a simple authorization ID or it " +
617                  "may be a value pattern to specify a range of " +
618                  "identities.  See " + ValuePattern.PUBLIC_JAVADOC_URL +
619                  " for complete details about the value pattern syntax.";
620    proxyAs = new StringArgument('Y', "proxyAs", false, 1, "{authzID}",
621                                 description);
622    proxyAs.addLongIdentifier("proxy-as");
623    parser.addArgument(proxyAs);
624
625    description = "Indicates that the client should operate in asynchronous " +
626                  "mode, in which it will not be necessary to wait for a " +
627                  "response to a previous request before sending the next " +
628                  "request.  Either the '--ratePerSecond' or the " +
629                  "'--maxOutstandingRequests' argument must be provided to " +
630                  "limit the number of outstanding requests.";
631    asynchronousMode = new BooleanArgument('a', "asynchronous", description);
632    parser.addArgument(asynchronousMode);
633
634    description = "Specifies the maximum number of outstanding requests " +
635                  "that should be allowed when operating in asynchronous mode.";
636    maxOutstandingRequests = new IntegerArgument('O', "maxOutstandingRequests",
637         false, 1, "{num}", description, 1, Integer.MAX_VALUE, (Integer) null);
638    maxOutstandingRequests.addLongIdentifier("max-outstanding-requests");
639    parser.addArgument(maxOutstandingRequests);
640
641    description = "Indicates that information about the result codes for " +
642                  "failed operations should not be displayed.";
643    suppressErrors = new BooleanArgument(null,
644         "suppressErrorResultCodes", 1, description);
645    suppressErrors.addLongIdentifier("suppress-error-result-codes");
646    parser.addArgument(suppressErrors);
647
648    description = "Generate output in CSV format rather than a " +
649                  "display-friendly format";
650    csvFormat = new BooleanArgument('c', "csv", 1, description);
651    parser.addArgument(csvFormat);
652
653    description = "Specifies the seed to use for the random number generator.";
654    randomSeed = new IntegerArgument('R', "randomSeed", false, 1, "{value}",
655         description);
656    randomSeed.addLongIdentifier("random-seed");
657    parser.addArgument(randomSeed);
658
659
660    parser.addDependentArgumentSet(asynchronousMode, ratePerSecond,
661         maxOutstandingRequests);
662    parser.addDependentArgumentSet(maxOutstandingRequests, asynchronousMode);
663  }
664
665
666
667  /**
668   * Indicates whether this tool supports creating connections to multiple
669   * servers.  If it is to support multiple servers, then the "--hostname" and
670   * "--port" arguments will be allowed to be provided multiple times, and
671   * will be required to be provided the same number of times.  The same type of
672   * communication security and bind credentials will be used for all servers.
673   *
674   * @return  {@code true} if this tool supports creating connections to
675   *          multiple servers, or {@code false} if not.
676   */
677  @Override()
678  protected boolean supportsMultipleServers()
679  {
680    return true;
681  }
682
683
684
685  /**
686   * Retrieves the connection options that should be used for connections
687   * created for use with this tool.
688   *
689   * @return  The connection options that should be used for connections created
690   *          for use with this tool.
691   */
692  @Override()
693  public LDAPConnectionOptions getConnectionOptions()
694  {
695    final LDAPConnectionOptions options = new LDAPConnectionOptions();
696    options.setUseSynchronousMode(! asynchronousMode.isPresent());
697    return options;
698  }
699
700
701
702  /**
703   * Performs the actual processing for this tool.  In this case, it gets a
704   * connection to the directory server and uses it to perform the requested
705   * searches.
706   *
707   * @return  The result code for the processing that was performed.
708   */
709  @Override()
710  public ResultCode doToolProcessing()
711  {
712    runningThread = Thread.currentThread();
713
714    try
715    {
716      return doToolProcessingInternal();
717    }
718    finally
719    {
720      runningThread = null;
721    }
722  }
723
724
725
726  /**
727   * Performs the actual processing for this tool.  In this case, it gets a
728   * connection to the directory server and uses it to perform the requested
729   * searches.
730   *
731   * @return  The result code for the processing that was performed.
732   */
733  private ResultCode doToolProcessingInternal()
734  {
735    // If the sample rate file argument was specified, then generate the sample
736    // variable rate data file and return.
737    if (sampleRateFile.isPresent())
738    {
739      try
740      {
741        RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue());
742        return ResultCode.SUCCESS;
743      }
744      catch (final Exception e)
745      {
746        debugException(e);
747        err("An error occurred while trying to write sample variable data " +
748             "rate file '", sampleRateFile.getValue().getAbsolutePath(),
749             "':  ", getExceptionMessage(e));
750        return ResultCode.LOCAL_ERROR;
751      }
752    }
753
754
755    // Determine the random seed to use.
756    final Long seed;
757    if (randomSeed.isPresent())
758    {
759      seed = Long.valueOf(randomSeed.getValue());
760    }
761    else
762    {
763      seed = null;
764    }
765
766    // Create value patterns for the base DN, filter, and proxied authorization
767    // DN.
768    final ValuePattern dnPattern;
769    try
770    {
771      dnPattern = new ValuePattern(baseDN.getValue(), seed);
772    }
773    catch (final ParseException pe)
774    {
775      debugException(pe);
776      err("Unable to parse the base DN value pattern:  ", pe.getMessage());
777      return ResultCode.PARAM_ERROR;
778    }
779
780    final ValuePattern filterPattern;
781    try
782    {
783      filterPattern = new ValuePattern(filter.getValue(), seed);
784    }
785    catch (final ParseException pe)
786    {
787      debugException(pe);
788      err("Unable to parse the filter pattern:  ", pe.getMessage());
789      return ResultCode.PARAM_ERROR;
790    }
791
792    final ValuePattern authzIDPattern;
793    if (proxyAs.isPresent())
794    {
795      try
796      {
797        authzIDPattern = new ValuePattern(proxyAs.getValue(), seed);
798      }
799      catch (final ParseException pe)
800      {
801        debugException(pe);
802        err("Unable to parse the proxied authorization pattern:  ",
803            pe.getMessage());
804        return ResultCode.PARAM_ERROR;
805      }
806    }
807    else
808    {
809      authzIDPattern = null;
810    }
811
812
813    // Get the attributes to return.
814    final String[] attrs;
815    if (attributes.isPresent())
816    {
817      final List<String> attrList = attributes.getValues();
818      attrs = new String[attrList.size()];
819      attrList.toArray(attrs);
820    }
821    else
822    {
823      attrs = NO_STRINGS;
824    }
825
826
827    // If the --ratePerSecond option was specified, then limit the rate
828    // accordingly.
829    FixedRateBarrier fixedRateBarrier = null;
830    if (ratePerSecond.isPresent() || variableRateData.isPresent())
831    {
832      // We might not have a rate per second if --variableRateData is specified.
833      // The rate typically doesn't matter except when we have warm-up
834      // intervals.  In this case, we'll run at the max rate.
835      final int intervalSeconds = collectionInterval.getValue();
836      final int ratePerInterval =
837           (ratePerSecond.getValue() == null)
838           ? Integer.MAX_VALUE
839           : ratePerSecond.getValue() * intervalSeconds;
840      fixedRateBarrier =
841           new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval);
842    }
843
844
845    // If --variableRateData was specified, then initialize a RateAdjustor.
846    RateAdjustor rateAdjustor = null;
847    if (variableRateData.isPresent())
848    {
849      try
850      {
851        rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier,
852             ratePerSecond.getValue(), variableRateData.getValue());
853      }
854      catch (final IOException e)
855      {
856        debugException(e);
857        err("Initializing the variable rates failed: " + e.getMessage());
858        return ResultCode.PARAM_ERROR;
859      }
860      catch (final IllegalArgumentException e)
861      {
862        debugException(e);
863        err("Initializing the variable rates failed: " + e.getMessage());
864        return ResultCode.PARAM_ERROR;
865      }
866    }
867
868
869    // If the --maxOutstandingRequests option was specified, then create the
870    // semaphore used to enforce that limit.
871    final Semaphore asyncSemaphore;
872    if (maxOutstandingRequests.isPresent())
873    {
874      asyncSemaphore = new Semaphore(maxOutstandingRequests.getValue());
875    }
876    else
877    {
878      asyncSemaphore = null;
879    }
880
881
882    // Determine whether to include timestamps in the output and if so what
883    // format should be used for them.
884    final boolean includeTimestamp;
885    final String timeFormat;
886    if (timestampFormat.getValue().equalsIgnoreCase("with-date"))
887    {
888      includeTimestamp = true;
889      timeFormat       = "dd/MM/yyyy HH:mm:ss";
890    }
891    else if (timestampFormat.getValue().equalsIgnoreCase("without-date"))
892    {
893      includeTimestamp = true;
894      timeFormat       = "HH:mm:ss";
895    }
896    else
897    {
898      includeTimestamp = false;
899      timeFormat       = null;
900    }
901
902
903    // Determine whether any warm-up intervals should be run.
904    final long totalIntervals;
905    final boolean warmUp;
906    int remainingWarmUpIntervals = warmUpIntervals.getValue();
907    if (remainingWarmUpIntervals > 0)
908    {
909      warmUp = true;
910      totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals;
911    }
912    else
913    {
914      warmUp = true;
915      totalIntervals = 0L + numIntervals.getValue();
916    }
917
918
919    // Create the table that will be used to format the output.
920    final OutputFormat outputFormat;
921    if (csvFormat.isPresent())
922    {
923      outputFormat = OutputFormat.CSV;
924    }
925    else
926    {
927      outputFormat = OutputFormat.COLUMNS;
928    }
929
930    final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp,
931         timeFormat, outputFormat, " ",
932         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
933                  "Searches/Sec"),
934         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
935                  "Avg Dur ms"),
936         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
937                  "Entries/Srch"),
938         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
939                  "Errors/Sec"),
940         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
941                  "Searches/Sec"),
942         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
943                  "Avg Dur ms"));
944
945
946    // Create values to use for statistics collection.
947    final AtomicLong        searchCounter   = new AtomicLong(0L);
948    final AtomicLong        entryCounter    = new AtomicLong(0L);
949    final AtomicLong        errorCounter    = new AtomicLong(0L);
950    final AtomicLong        searchDurations = new AtomicLong(0L);
951    final ResultCodeCounter rcCounter       = new ResultCodeCounter();
952
953
954    // Determine the length of each interval in milliseconds.
955    final long intervalMillis = 1000L * collectionInterval.getValue();
956
957
958    // Create the threads to use for the searches.
959    final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1);
960    final SearchRateThread[] threads =
961         new SearchRateThread[numThreads.getValue()];
962    for (int i=0; i < threads.length; i++)
963    {
964      final LDAPConnection connection;
965      try
966      {
967        connection = getConnection();
968      }
969      catch (final LDAPException le)
970      {
971        debugException(le);
972        err("Unable to connect to the directory server:  ",
973            getExceptionMessage(le));
974        return le.getResultCode();
975      }
976
977      threads[i] = new SearchRateThread(this, i, connection,
978           asynchronousMode.isPresent(), dnPattern, scopeArg.getValue(),
979           filterPattern, attrs, authzIDPattern,
980           iterationsBeforeReconnect.getValue(), barrier, searchCounter,
981           entryCounter, searchDurations, errorCounter, rcCounter,
982           fixedRateBarrier, asyncSemaphore);
983      threads[i].start();
984    }
985
986
987    // Display the table header.
988    for (final String headerLine : formatter.getHeaderLines(true))
989    {
990      out(headerLine);
991    }
992
993
994    // Start the RateAdjustor before the threads so that the initial value is
995    // in place before any load is generated unless we're doing a warm-up in
996    // which case, we'll start it after the warm-up is complete.
997    if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0))
998    {
999      rateAdjustor.start();
1000    }
1001
1002
1003    // Indicate that the threads can start running.
1004    try
1005    {
1006      barrier.await();
1007    }
1008    catch (final Exception e)
1009    {
1010      debugException(e);
1011    }
1012
1013    long overallStartTime = System.nanoTime();
1014    long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis;
1015
1016
1017    boolean setOverallStartTime = false;
1018    long    lastDuration        = 0L;
1019    long    lastNumEntries      = 0L;
1020    long    lastNumErrors       = 0L;
1021    long    lastNumSearches     = 0L;
1022    long    lastEndTime         = System.nanoTime();
1023    for (long i=0; i < totalIntervals; i++)
1024    {
1025      if (rateAdjustor != null)
1026      {
1027        if (! rateAdjustor.isAlive())
1028        {
1029          out("All of the rates in " + variableRateData.getValue().getName() +
1030              " have been completed.");
1031          break;
1032        }
1033      }
1034
1035      final long startTimeMillis = System.currentTimeMillis();
1036      final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis;
1037      nextIntervalStartTime += intervalMillis;
1038      if (sleepTimeMillis > 0)
1039      {
1040        sleeper.sleep(sleepTimeMillis);
1041      }
1042
1043      if (stopRequested.get())
1044      {
1045        break;
1046      }
1047
1048      final long endTime          = System.nanoTime();
1049      final long intervalDuration = endTime - lastEndTime;
1050
1051      final long numSearches;
1052      final long numEntries;
1053      final long numErrors;
1054      final long totalDuration;
1055      if (warmUp && (remainingWarmUpIntervals > 0))
1056      {
1057        numSearches   = searchCounter.getAndSet(0L);
1058        numEntries    = entryCounter.getAndSet(0L);
1059        numErrors     = errorCounter.getAndSet(0L);
1060        totalDuration = searchDurations.getAndSet(0L);
1061      }
1062      else
1063      {
1064        numSearches   = searchCounter.get();
1065        numEntries    = entryCounter.get();
1066        numErrors     = errorCounter.get();
1067        totalDuration = searchDurations.get();
1068      }
1069
1070      final long recentNumSearches = numSearches - lastNumSearches;
1071      final long recentNumEntries = numEntries - lastNumEntries;
1072      final long recentNumErrors = numErrors - lastNumErrors;
1073      final long recentDuration = totalDuration - lastDuration;
1074
1075      final double numSeconds = intervalDuration / 1000000000.0d;
1076      final double recentSearchRate = recentNumSearches / numSeconds;
1077      final double recentErrorRate  = recentNumErrors / numSeconds;
1078
1079      final double recentAvgDuration;
1080      final double recentEntriesPerSearch;
1081      if (recentNumSearches > 0L)
1082      {
1083        recentEntriesPerSearch = 1.0d * recentNumEntries / recentNumSearches;
1084        recentAvgDuration = 1.0d * recentDuration / recentNumSearches / 1000000;
1085      }
1086      else
1087      {
1088        recentEntriesPerSearch = 0.0d;
1089        recentAvgDuration = 0.0d;
1090      }
1091
1092
1093      if (warmUp && (remainingWarmUpIntervals > 0))
1094      {
1095        out(formatter.formatRow(recentSearchRate, recentAvgDuration,
1096             recentEntriesPerSearch, recentErrorRate, "warming up",
1097             "warming up"));
1098
1099        remainingWarmUpIntervals--;
1100        if (remainingWarmUpIntervals == 0)
1101        {
1102          out("Warm-up completed.  Beginning overall statistics collection.");
1103          setOverallStartTime = true;
1104          if (rateAdjustor != null)
1105          {
1106            rateAdjustor.start();
1107          }
1108        }
1109      }
1110      else
1111      {
1112        if (setOverallStartTime)
1113        {
1114          overallStartTime    = lastEndTime;
1115          setOverallStartTime = false;
1116        }
1117
1118        final double numOverallSeconds =
1119             (endTime - overallStartTime) / 1000000000.0d;
1120        final double overallSearchRate = numSearches / numOverallSeconds;
1121
1122        final double overallAvgDuration;
1123        if (numSearches > 0L)
1124        {
1125          overallAvgDuration = 1.0d * totalDuration / numSearches / 1000000;
1126        }
1127        else
1128        {
1129          overallAvgDuration = 0.0d;
1130        }
1131
1132        out(formatter.formatRow(recentSearchRate, recentAvgDuration,
1133             recentEntriesPerSearch, recentErrorRate, overallSearchRate,
1134             overallAvgDuration));
1135
1136        lastNumSearches = numSearches;
1137        lastNumEntries  = numEntries;
1138        lastNumErrors   = numErrors;
1139        lastDuration    = totalDuration;
1140      }
1141
1142      final List<ObjectPair<ResultCode,Long>> rcCounts =
1143           rcCounter.getCounts(true);
1144      if ((! suppressErrors.isPresent()) && (! rcCounts.isEmpty()))
1145      {
1146        err("\tError Results:");
1147        for (final ObjectPair<ResultCode,Long> p : rcCounts)
1148        {
1149          err("\t", p.getFirst().getName(), ":  ", p.getSecond());
1150        }
1151      }
1152
1153      lastEndTime = endTime;
1154    }
1155
1156
1157    // Shut down the RateAdjustor if we have one.
1158    if (rateAdjustor != null)
1159    {
1160      rateAdjustor.shutDown();
1161    }
1162
1163
1164    // Stop all of the threads.
1165    ResultCode resultCode = ResultCode.SUCCESS;
1166    for (final SearchRateThread t : threads)
1167    {
1168      t.signalShutdown();
1169    }
1170    for (final SearchRateThread t : threads)
1171    {
1172      final ResultCode r = t.waitForShutdown();
1173      if (resultCode == ResultCode.SUCCESS)
1174      {
1175        resultCode = r;
1176      }
1177    }
1178
1179    return resultCode;
1180  }
1181
1182
1183
1184  /**
1185   * Requests that this tool stop running.  This method will attempt to wait
1186   * for all threads to complete before returning control to the caller.
1187   */
1188  public void stopRunning()
1189  {
1190    stopRequested.set(true);
1191    sleeper.wakeup();
1192
1193    final Thread t = runningThread;
1194    if (t != null)
1195    {
1196      try
1197      {
1198        t.join();
1199      }
1200      catch (final Exception e)
1201      {
1202        debugException(e);
1203      }
1204    }
1205  }
1206
1207
1208
1209  /**
1210   * Retrieves the maximum number of outstanding requests that may be in
1211   * progress at any time, if appropriate.
1212   *
1213   * @return  The maximum number of outstanding requests that may be in progress
1214   *          at any time, or -1 if the tool was not configured to perform
1215   *          asynchronous searches with a maximum number of outstanding
1216   *          requests.
1217   */
1218  int getMaxOutstandingRequests()
1219  {
1220    if (maxOutstandingRequests.isPresent())
1221    {
1222      return maxOutstandingRequests.getValue();
1223    }
1224    else
1225    {
1226      return -1;
1227    }
1228  }
1229
1230
1231
1232  /**
1233   * {@inheritDoc}
1234   */
1235  @Override()
1236  public LinkedHashMap<String[],String> getExampleUsages()
1237  {
1238    final LinkedHashMap<String[],String> examples =
1239         new LinkedHashMap<String[],String>(2);
1240
1241    String[] args =
1242    {
1243      "--hostname", "server.example.com",
1244      "--port", "389",
1245      "--bindDN", "uid=admin,dc=example,dc=com",
1246      "--bindPassword", "password",
1247      "--baseDN", "dc=example,dc=com",
1248      "--scope", "sub",
1249      "--filter", "(uid=user.[1-1000000])",
1250      "--attribute", "givenName",
1251      "--attribute", "sn",
1252      "--attribute", "mail",
1253      "--numThreads", "10"
1254    };
1255    String description =
1256         "Test search performance by searching randomly across a set " +
1257         "of one million users located below 'dc=example,dc=com' with ten " +
1258         "concurrent threads.  The entries returned to the client will " +
1259         "include the givenName, sn, and mail attributes.";
1260    examples.put(args, description);
1261
1262    args = new String[]
1263    {
1264      "--generateSampleRateFile", "variable-rate-data.txt"
1265    };
1266    description =
1267         "Generate a sample variable rate definition file that may be used " +
1268         "in conjunction with the --variableRateData argument.  The sample " +
1269         "file will include comments that describe the format for data to be " +
1270         "included in this file.";
1271    examples.put(args, description);
1272
1273    return examples;
1274  }
1275}