001/* 002 * Copyright 2010-2016 UnboundID Corp. 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright (C) 2010-2016 UnboundID Corp. 007 * 008 * This program is free software; you can redistribute it and/or modify 009 * it under the terms of the GNU General Public License (GPLv2 only) 010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 011 * as published by the Free Software Foundation. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Public License for more details. 017 * 018 * You should have received a copy of the GNU General Public License 019 * along with this program; if not, see <http://www.gnu.org/licenses>. 020 */ 021package com.unboundid.ldap.sdk.examples; 022 023 024 025import java.io.IOException; 026import java.io.OutputStream; 027import java.io.Serializable; 028import java.text.ParseException; 029import java.util.LinkedHashMap; 030import java.util.LinkedHashSet; 031import java.util.List; 032import java.util.Random; 033import java.util.concurrent.CyclicBarrier; 034import java.util.concurrent.atomic.AtomicBoolean; 035import java.util.concurrent.atomic.AtomicLong; 036 037import com.unboundid.ldap.sdk.LDAPConnection; 038import com.unboundid.ldap.sdk.LDAPConnectionOptions; 039import com.unboundid.ldap.sdk.LDAPException; 040import com.unboundid.ldap.sdk.ResultCode; 041import com.unboundid.ldap.sdk.SearchScope; 042import com.unboundid.ldap.sdk.Version; 043import com.unboundid.util.ColumnFormatter; 044import com.unboundid.util.FixedRateBarrier; 045import com.unboundid.util.FormattableColumn; 046import com.unboundid.util.HorizontalAlignment; 047import com.unboundid.util.LDAPCommandLineTool; 048import com.unboundid.util.ObjectPair; 049import com.unboundid.util.OutputFormat; 050import com.unboundid.util.RateAdjustor; 051import com.unboundid.util.ResultCodeCounter; 052import com.unboundid.util.ThreadSafety; 053import com.unboundid.util.ThreadSafetyLevel; 054import com.unboundid.util.ValuePattern; 055import com.unboundid.util.WakeableSleeper; 056import com.unboundid.util.args.ArgumentException; 057import com.unboundid.util.args.ArgumentParser; 058import com.unboundid.util.args.BooleanArgument; 059import com.unboundid.util.args.FileArgument; 060import com.unboundid.util.args.IntegerArgument; 061import com.unboundid.util.args.ScopeArgument; 062import com.unboundid.util.args.StringArgument; 063 064import static com.unboundid.util.Debug.*; 065import static com.unboundid.util.StaticUtils.*; 066 067 068 069/** 070 * This class provides a tool that can be used to search an LDAP directory 071 * server repeatedly using multiple threads, and then modify each entry 072 * returned by that server. It can help provide an estimate of the combined 073 * search and modify performance that a directory server is able to achieve. 074 * Either or both of the base DN and the search filter may be a value pattern as 075 * described in the {@link ValuePattern} class. This makes it possible to 076 * search over a range of entries rather than repeatedly performing searches 077 * with the same base DN and filter. 078 * <BR><BR> 079 * Some of the APIs demonstrated by this example include: 080 * <UL> 081 * <LI>Argument Parsing (from the {@code com.unboundid.util.args} 082 * package)</LI> 083 * <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util} 084 * package)</LI> 085 * <LI>LDAP Communication (from the {@code com.unboundid.ldap.sdk} 086 * package)</LI> 087 * <LI>Value Patterns (from the {@code com.unboundid.util} package)</LI> 088 * </UL> 089 * <BR><BR> 090 * All of the necessary information is provided using command line arguments. 091 * Supported arguments include those allowed by the {@link LDAPCommandLineTool} 092 * class, as well as the following additional arguments: 093 * <UL> 094 * <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use 095 * for the searches. This must be provided. It may be a simple DN, or it 096 * may be a value pattern to express a range of base DNs.</LI> 097 * <LI>"-s {scope}" or "--scope {scope}" -- specifies the scope to use for the 098 * search. The scope value should be one of "base", "one", "sub", or 099 * "subord". If this isn't specified, then a scope of "sub" will be 100 * used.</LI> 101 * <LI>"-f {filter}" or "--filter {filter}" -- specifies the filter to use for 102 * the searches. This must be provided. It may be a simple filter, or it 103 * may be a value pattern to express a range of filters.</LI> 104 * <LI>"-A {name}" or "--attribute {name}" -- specifies the name of an 105 * attribute that should be included in entries returned from the server. 106 * If this is not provided, then all user attributes will be requested. 107 * This may include special tokens that the server may interpret, like 108 * "1.1" to indicate that no attributes should be returned, "*", for all 109 * user attributes, or "+" for all operational attributes. Multiple 110 * attributes may be requested with multiple instances of this 111 * argument.</LI> 112 * <LI>"-m {name}" or "--modifyAttribute {name}" -- specifies the name of the 113 * attribute to modify. Multiple attributes may be modified by providing 114 * multiple instances of this argument. At least one attribute must be 115 * provided.</LI> 116 * <LI>"-l {num}" or "--valueLength {num}" -- specifies the length in bytes to 117 * use for the values of the target attributes to modify. If this is not 118 * provided, then a default length of 10 bytes will be used.</LI> 119 * <LI>"-C {chars}" or "--characterSet {chars}" -- specifies the set of 120 * characters that will be used to generate the values to use for the 121 * target attributes to modify. It should only include ASCII characters. 122 * Values will be generated from randomly-selected characters from this 123 * set. If this is not provided, then a default set of lowercase 124 * alphabetic characters will be used.</LI> 125 * <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of 126 * concurrent threads to use when performing the searches. If this is not 127 * provided, then a default of one thread will be used.</LI> 128 * <LI>"-i {sec}" or "--intervalDuration {sec}" -- specifies the length of 129 * time in seconds between lines out output. If this is not provided, 130 * then a default interval duration of five seconds will be used.</LI> 131 * <LI>"-I {num}" or "--numIntervals {num}" -- specifies the maximum number of 132 * intervals for which to run. If this is not provided, then it will 133 * run forever.</LI> 134 * <LI>"--iterationsBeforeReconnect {num}" -- specifies the number of search 135 * iterations that should be performed on a connection before that 136 * connection is closed and replaced with a newly-established (and 137 * authenticated, if appropriate) connection.</LI> 138 * <LI>"-r {ops-per-second}" or "--ratePerSecond {ops-per-second}" -- 139 * specifies the target number of operations to perform per second. Each 140 * search and modify operation will be counted separately for this 141 * purpose, so if a value of 1 is specified and a search returns two 142 * entries, then a total of three seconds will be required (one for the 143 * search and one for the modify for each entry). It is still necessary 144 * to specify a sufficient number of threads for achieving this rate. If 145 * this option is not provided, then the tool will run at the maximum rate 146 * for the specified number of threads.</LI> 147 * <LI>"--variableRateData {path}" -- specifies the path to a file containing 148 * information needed to allow the tool to vary the target rate over time. 149 * If this option is not provided, then the tool will either use a fixed 150 * target rate as specified by the "--ratePerSecond" argument, or it will 151 * run at the maximum rate.</LI> 152 * <LI>"--generateSampleRateFile {path}" -- specifies the path to a file to 153 * which sample data will be written illustrating and describing the 154 * format of the file expected to be used in conjunction with the 155 * "--variableRateData" argument.</LI> 156 * <LI>"--warmUpIntervals {num}" -- specifies the number of intervals to 157 * complete before beginning overall statistics collection.</LI> 158 * <LI>"--timestampFormat {format}" -- specifies the format to use for 159 * timestamps included before each output line. The format may be one of 160 * "none" (for no timestamps), "with-date" (to include both the date and 161 * the time), or "without-date" (to include only time time).</LI> 162 * <LI>"-Y {authzID}" or "--proxyAs {authzID}" -- Use the proxied 163 * authorization v2 control to request that the operations be processed 164 * using an alternate authorization identity. In this case, the bind DN 165 * should be that of a user that has permission to use this control. The 166 * authorization identity may be a value pattern.</LI> 167 * <LI>"--suppressErrorResultCodes" -- Indicates that information about the 168 * result codes for failed operations should not be displayed.</LI> 169 * <LI>"-c" or "--csv" -- Generate output in CSV format rather than a 170 * display-friendly format.</LI> 171 * </UL> 172 */ 173@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 174public final class SearchAndModRate 175 extends LDAPCommandLineTool 176 implements Serializable 177{ 178 /** 179 * The serial version UID for this serializable class. 180 */ 181 private static final long serialVersionUID = 3242469381380526294L; 182 183 184 185 // Indicates whether a request has been made to stop running. 186 private final AtomicBoolean stopRequested; 187 188 // The argument used to indicate whether to generate output in CSV format. 189 private BooleanArgument csvFormat; 190 191 // The argument used to indicate whether to suppress information about error 192 // result codes. 193 private BooleanArgument suppressErrors; 194 195 // The argument used to specify the collection interval. 196 private IntegerArgument collectionInterval; 197 198 // The argument used to specify the number of search and modify iterations on 199 // a connection before it is closed and re-established. 200 private IntegerArgument iterationsBeforeReconnect; 201 202 // The argument used to specify the number of intervals. 203 private IntegerArgument numIntervals; 204 205 // The argument used to specify the number of threads. 206 private IntegerArgument numThreads; 207 208 // The argument used to specify the seed to use for the random number 209 // generator. 210 private IntegerArgument randomSeed; 211 212 // The target rate of operations per second. 213 private IntegerArgument ratePerSecond; 214 215 // The argument used to specify a variable rate file. 216 private FileArgument sampleRateFile; 217 218 // The argument used to specify a variable rate file. 219 private FileArgument variableRateData; 220 221 // The argument used to specify the length of the values to generate. 222 private IntegerArgument valueLength; 223 224 // The number of warm-up intervals to perform. 225 private IntegerArgument warmUpIntervals; 226 227 // The argument used to specify the scope for the searches. 228 private ScopeArgument scopeArg; 229 230 // The argument used to specify the base DNs for the searches. 231 private StringArgument baseDN; 232 233 // The argument used to specify the set of characters to use when generating 234 // values. 235 private StringArgument characterSet; 236 237 // The argument used to specify the filters for the searches. 238 private StringArgument filter; 239 240 // The argument used to specify the attributes to modify. 241 private StringArgument modifyAttributes; 242 243 // The argument used to specify the proxied authorization identity. 244 private StringArgument proxyAs; 245 246 // The argument used to specify the attributes to return. 247 private StringArgument returnAttributes; 248 249 // The argument used to specify the timestamp format. 250 private StringArgument timestampFormat; 251 252 // The thread currently being used to run the searchrate tool. 253 private volatile Thread runningThread; 254 255 // A wakeable sleeper that will be used to sleep between reporting intervals. 256 private final WakeableSleeper sleeper; 257 258 259 260 /** 261 * Parse the provided command line arguments and make the appropriate set of 262 * changes. 263 * 264 * @param args The command line arguments provided to this program. 265 */ 266 public static void main(final String[] args) 267 { 268 final ResultCode resultCode = main(args, System.out, System.err); 269 if (resultCode != ResultCode.SUCCESS) 270 { 271 System.exit(resultCode.intValue()); 272 } 273 } 274 275 276 277 /** 278 * Parse the provided command line arguments and make the appropriate set of 279 * changes. 280 * 281 * @param args The command line arguments provided to this program. 282 * @param outStream The output stream to which standard out should be 283 * written. It may be {@code null} if output should be 284 * suppressed. 285 * @param errStream The output stream to which standard error should be 286 * written. It may be {@code null} if error messages 287 * should be suppressed. 288 * 289 * @return A result code indicating whether the processing was successful. 290 */ 291 public static ResultCode main(final String[] args, 292 final OutputStream outStream, 293 final OutputStream errStream) 294 { 295 final SearchAndModRate searchAndModRate = 296 new SearchAndModRate(outStream, errStream); 297 return searchAndModRate.runTool(args); 298 } 299 300 301 302 /** 303 * Creates a new instance of this tool. 304 * 305 * @param outStream The output stream to which standard out should be 306 * written. It may be {@code null} if output should be 307 * suppressed. 308 * @param errStream The output stream to which standard error should be 309 * written. It may be {@code null} if error messages 310 * should be suppressed. 311 */ 312 public SearchAndModRate(final OutputStream outStream, 313 final OutputStream errStream) 314 { 315 super(outStream, errStream); 316 317 stopRequested = new AtomicBoolean(false); 318 sleeper = new WakeableSleeper(); 319 } 320 321 322 323 /** 324 * Retrieves the name for this tool. 325 * 326 * @return The name for this tool. 327 */ 328 @Override() 329 public String getToolName() 330 { 331 return "search-and-mod-rate"; 332 } 333 334 335 336 /** 337 * Retrieves the description for this tool. 338 * 339 * @return The description for this tool. 340 */ 341 @Override() 342 public String getToolDescription() 343 { 344 return "Perform repeated searches against an " + 345 "LDAP directory server and modify each entry returned."; 346 } 347 348 349 350 /** 351 * Retrieves the version string for this tool. 352 * 353 * @return The version string for this tool. 354 */ 355 @Override() 356 public String getToolVersion() 357 { 358 return Version.NUMERIC_VERSION_STRING; 359 } 360 361 362 363 /** 364 * Indicates whether this tool should provide support for an interactive mode, 365 * in which the tool offers a mode in which the arguments can be provided in 366 * a text-driven menu rather than requiring them to be given on the command 367 * line. If interactive mode is supported, it may be invoked using the 368 * "--interactive" argument. Alternately, if interactive mode is supported 369 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then 370 * interactive mode may be invoked by simply launching the tool without any 371 * arguments. 372 * 373 * @return {@code true} if this tool supports interactive mode, or 374 * {@code false} if not. 375 */ 376 @Override() 377 public boolean supportsInteractiveMode() 378 { 379 return true; 380 } 381 382 383 384 /** 385 * Indicates whether this tool defaults to launching in interactive mode if 386 * the tool is invoked without any command-line arguments. This will only be 387 * used if {@link #supportsInteractiveMode()} returns {@code true}. 388 * 389 * @return {@code true} if this tool defaults to using interactive mode if 390 * launched without any command-line arguments, or {@code false} if 391 * not. 392 */ 393 @Override() 394 public boolean defaultsToInteractiveMode() 395 { 396 return true; 397 } 398 399 400 401 /** 402 * Indicates whether this tool should provide arguments for redirecting output 403 * to a file. If this method returns {@code true}, then the tool will offer 404 * an "--outputFile" argument that will specify the path to a file to which 405 * all standard output and standard error content will be written, and it will 406 * also offer a "--teeToStandardOut" argument that can only be used if the 407 * "--outputFile" argument is present and will cause all output to be written 408 * to both the specified output file and to standard output. 409 * 410 * @return {@code true} if this tool should provide arguments for redirecting 411 * output to a file, or {@code false} if not. 412 */ 413 @Override() 414 protected boolean supportsOutputFile() 415 { 416 return true; 417 } 418 419 420 421 /** 422 * Indicates whether this tool should default to interactively prompting for 423 * the bind password if a password is required but no argument was provided 424 * to indicate how to get the password. 425 * 426 * @return {@code true} if this tool should default to interactively 427 * prompting for the bind password, or {@code false} if not. 428 */ 429 @Override() 430 protected boolean defaultToPromptForBindPassword() 431 { 432 return true; 433 } 434 435 436 437 /** 438 * Indicates whether this tool supports the use of a properties file for 439 * specifying default values for arguments that aren't specified on the 440 * command line. 441 * 442 * @return {@code true} if this tool supports the use of a properties file 443 * for specifying default values for arguments that aren't specified 444 * on the command line, or {@code false} if not. 445 */ 446 @Override() 447 public boolean supportsPropertiesFile() 448 { 449 return true; 450 } 451 452 453 454 /** 455 * Indicates whether the LDAP-specific arguments should include alternate 456 * versions of all long identifiers that consist of multiple words so that 457 * they are available in both camelCase and dash-separated versions. 458 * 459 * @return {@code true} if this tool should provide multiple versions of 460 * long identifiers for LDAP-specific arguments, or {@code false} if 461 * not. 462 */ 463 @Override() 464 protected boolean includeAlternateLongIdentifiers() 465 { 466 return true; 467 } 468 469 470 471 /** 472 * Adds the arguments used by this program that aren't already provided by the 473 * generic {@code LDAPCommandLineTool} framework. 474 * 475 * @param parser The argument parser to which the arguments should be added. 476 * 477 * @throws ArgumentException If a problem occurs while adding the arguments. 478 */ 479 @Override() 480 public void addNonLDAPArguments(final ArgumentParser parser) 481 throws ArgumentException 482 { 483 String description = "The base DN to use for the searches. It may be a " + 484 "simple DN or a value pattern to specify a range of DNs (e.g., " + 485 "\"uid=user.[1-1000],ou=People,dc=example,dc=com\"). See " + 486 ValuePattern.PUBLIC_JAVADOC_URL + " for complete details about the " + 487 "value pattern syntax. This must be provided."; 488 baseDN = new StringArgument('b', "baseDN", true, 1, "{dn}", description); 489 baseDN.setArgumentGroupName("Search And Modification Arguments"); 490 baseDN.addLongIdentifier("base-dn"); 491 parser.addArgument(baseDN); 492 493 494 description = "The scope to use for the searches. It should be 'base', " + 495 "'one', 'sub', or 'subord'. If this is not provided, then " + 496 "a default scope of 'sub' will be used."; 497 scopeArg = new ScopeArgument('s', "scope", false, "{scope}", description, 498 SearchScope.SUB); 499 scopeArg.setArgumentGroupName("Search And Modification Arguments"); 500 parser.addArgument(scopeArg); 501 502 503 description = "The filter to use for the searches. It may be a simple " + 504 "filter or a value pattern to specify a range of filters " + 505 "(e.g., \"(uid=user.[1-1000])\"). See " + 506 ValuePattern.PUBLIC_JAVADOC_URL + " for complete details " + 507 "about the value pattern syntax. This must be provided."; 508 filter = new StringArgument('f', "filter", true, 1, "{filter}", 509 description); 510 filter.setArgumentGroupName("Search And Modification Arguments"); 511 parser.addArgument(filter); 512 513 514 description = "The name of an attribute to include in entries returned " + 515 "from the searches. Multiple attributes may be requested " + 516 "by providing this argument multiple times. If no request " + 517 "attributes are provided, then the entries returned will " + 518 "include all user attributes."; 519 returnAttributes = new StringArgument('A', "attribute", false, 0, "{name}", 520 description); 521 returnAttributes.setArgumentGroupName("Search And Modification Arguments"); 522 parser.addArgument(returnAttributes); 523 524 525 description = "The name of the attribute to modify. Multiple attributes " + 526 "may be specified by providing this argument multiple " + 527 "times. At least one attribute must be specified."; 528 modifyAttributes = new StringArgument('m', "modifyAttribute", true, 0, 529 "{name}", description); 530 modifyAttributes.setArgumentGroupName("Search And Modification Arguments"); 531 modifyAttributes.addLongIdentifier("modify-attribute"); 532 parser.addArgument(modifyAttributes); 533 534 535 description = "The length in bytes to use when generating values for the " + 536 "modifications. If this is not provided, then a default " + 537 "length of ten bytes will be used."; 538 valueLength = new IntegerArgument('l', "valueLength", true, 1, "{num}", 539 description, 1, Integer.MAX_VALUE, 10); 540 valueLength.setArgumentGroupName("Search And Modification Arguments"); 541 valueLength.addLongIdentifier("value-length"); 542 parser.addArgument(valueLength); 543 544 545 description = "The set of characters to use to generate the values for " + 546 "the modifications. It should only include ASCII " + 547 "characters. If this is not provided, then a default set " + 548 "of lowercase alphabetic characters will be used."; 549 characterSet = new StringArgument('C', "characterSet", true, 1, "{chars}", 550 description, 551 "abcdefghijklmnopqrstuvwxyz"); 552 characterSet.setArgumentGroupName("Search And Modification Arguments"); 553 characterSet.addLongIdentifier("character-set"); 554 parser.addArgument(characterSet); 555 556 557 description = "The number of threads to use to perform the searches. If " + 558 "this is not provided, then a default of one thread will " + 559 "be used."; 560 numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}", 561 description, 1, Integer.MAX_VALUE, 1); 562 numThreads.setArgumentGroupName("Rate Management Arguments"); 563 numThreads.addLongIdentifier("num-threads"); 564 parser.addArgument(numThreads); 565 566 567 description = "The length of time in seconds between output lines. If " + 568 "this is not provided, then a default interval of five " + 569 "seconds will be used."; 570 collectionInterval = new IntegerArgument('i', "intervalDuration", true, 1, 571 "{num}", description, 1, 572 Integer.MAX_VALUE, 5); 573 collectionInterval.setArgumentGroupName("Rate Management Arguments"); 574 collectionInterval.addLongIdentifier("interval-duration"); 575 parser.addArgument(collectionInterval); 576 577 578 description = "The maximum number of intervals for which to run. If " + 579 "this is not provided, then the tool will run until it is " + 580 "interrupted."; 581 numIntervals = new IntegerArgument('I', "numIntervals", true, 1, "{num}", 582 description, 1, Integer.MAX_VALUE, 583 Integer.MAX_VALUE); 584 numIntervals.setArgumentGroupName("Rate Management Arguments"); 585 numIntervals.addLongIdentifier("num-intervals"); 586 parser.addArgument(numIntervals); 587 588 description = "The number of search and modify iterations that should be " + 589 "processed on a connection before that connection is " + 590 "closed and replaced with a newly-established (and " + 591 "authenticated, if appropriate) connection. If this is " + 592 "not provided, then connections will not be periodically " + 593 "closed and re-established."; 594 iterationsBeforeReconnect = new IntegerArgument(null, 595 "iterationsBeforeReconnect", false, 1, "{num}", description, 0); 596 iterationsBeforeReconnect.setArgumentGroupName("Rate Management Arguments"); 597 iterationsBeforeReconnect.addLongIdentifier("iterations-before-reconnect"); 598 parser.addArgument(iterationsBeforeReconnect); 599 600 description = "The target number of searches to perform per second. It " + 601 "is still necessary to specify a sufficient number of " + 602 "threads for achieving this rate. If neither this option " + 603 "nor --variableRateData is provided, then the tool will " + 604 "run at the maximum rate for the specified number of " + 605 "threads."; 606 ratePerSecond = new IntegerArgument('r', "ratePerSecond", false, 1, 607 "{searches-per-second}", description, 608 1, Integer.MAX_VALUE); 609 ratePerSecond.setArgumentGroupName("Rate Management Arguments"); 610 ratePerSecond.addLongIdentifier("rate-per-second"); 611 parser.addArgument(ratePerSecond); 612 613 final String variableRateDataArgName = "variableRateData"; 614 final String generateSampleRateFileArgName = "generateSampleRateFile"; 615 description = RateAdjustor.getVariableRateDataArgumentDescription( 616 generateSampleRateFileArgName); 617 variableRateData = new FileArgument(null, variableRateDataArgName, false, 1, 618 "{path}", description, true, true, true, 619 false); 620 variableRateData.setArgumentGroupName("Rate Management Arguments"); 621 variableRateData.addLongIdentifier("variable-rate-data"); 622 parser.addArgument(variableRateData); 623 624 description = RateAdjustor.getGenerateSampleVariableRateFileDescription( 625 variableRateDataArgName); 626 sampleRateFile = new FileArgument(null, generateSampleRateFileArgName, 627 false, 1, "{path}", description, false, 628 true, true, false); 629 sampleRateFile.setArgumentGroupName("Rate Management Arguments"); 630 sampleRateFile.addLongIdentifier("generate-sample-rate-file"); 631 sampleRateFile.setUsageArgument(true); 632 parser.addArgument(sampleRateFile); 633 parser.addExclusiveArgumentSet(variableRateData, sampleRateFile); 634 635 description = "The number of intervals to complete before beginning " + 636 "overall statistics collection. Specifying a nonzero " + 637 "number of warm-up intervals gives the client and server " + 638 "a chance to warm up without skewing performance results."; 639 warmUpIntervals = new IntegerArgument(null, "warmUpIntervals", true, 1, 640 "{num}", description, 0, Integer.MAX_VALUE, 0); 641 warmUpIntervals.setArgumentGroupName("Rate Management Arguments"); 642 warmUpIntervals.addLongIdentifier("warm-up-intervals"); 643 parser.addArgument(warmUpIntervals); 644 645 description = "Indicates the format to use for timestamps included in " + 646 "the output. A value of 'none' indicates that no " + 647 "timestamps should be included. A value of 'with-date' " + 648 "indicates that both the date and the time should be " + 649 "included. A value of 'without-date' indicates that only " + 650 "the time should be included."; 651 final LinkedHashSet<String> allowedFormats = new LinkedHashSet<String>(3); 652 allowedFormats.add("none"); 653 allowedFormats.add("with-date"); 654 allowedFormats.add("without-date"); 655 timestampFormat = new StringArgument(null, "timestampFormat", true, 1, 656 "{format}", description, allowedFormats, "none"); 657 timestampFormat.addLongIdentifier("timestamp-format"); 658 parser.addArgument(timestampFormat); 659 660 description = "Indicates that the proxied authorization control (as " + 661 "defined in RFC 4370) should be used to request that " + 662 "operations be processed using an alternate authorization " + 663 "identity. This may be a simple authorization ID or it " + 664 "may be a value pattern to specify a range of " + 665 "identities. See " + ValuePattern.PUBLIC_JAVADOC_URL + 666 " for complete details about the value pattern syntax."; 667 proxyAs = new StringArgument('Y', "proxyAs", false, 1, "{authzID}", 668 description); 669 proxyAs.addLongIdentifier("proxy-as"); 670 parser.addArgument(proxyAs); 671 672 description = "Indicates that information about the result codes for " + 673 "failed operations should not be displayed."; 674 suppressErrors = new BooleanArgument(null, 675 "suppressErrorResultCodes", 1, description); 676 suppressErrors.addLongIdentifier("suppress-error-result-codes"); 677 parser.addArgument(suppressErrors); 678 679 description = "Generate output in CSV format rather than a " + 680 "display-friendly format"; 681 csvFormat = new BooleanArgument('c', "csv", 1, description); 682 parser.addArgument(csvFormat); 683 684 description = "Specifies the seed to use for the random number generator."; 685 randomSeed = new IntegerArgument('R', "randomSeed", false, 1, "{value}", 686 description); 687 randomSeed.addLongIdentifier("random-seed"); 688 parser.addArgument(randomSeed); 689 } 690 691 692 693 /** 694 * Indicates whether this tool supports creating connections to multiple 695 * servers. If it is to support multiple servers, then the "--hostname" and 696 * "--port" arguments will be allowed to be provided multiple times, and 697 * will be required to be provided the same number of times. The same type of 698 * communication security and bind credentials will be used for all servers. 699 * 700 * @return {@code true} if this tool supports creating connections to 701 * multiple servers, or {@code false} if not. 702 */ 703 @Override() 704 protected boolean supportsMultipleServers() 705 { 706 return true; 707 } 708 709 710 711 /** 712 * Retrieves the connection options that should be used for connections 713 * created for use with this tool. 714 * 715 * @return The connection options that should be used for connections created 716 * for use with this tool. 717 */ 718 @Override() 719 public LDAPConnectionOptions getConnectionOptions() 720 { 721 final LDAPConnectionOptions options = new LDAPConnectionOptions(); 722 options.setUseSynchronousMode(true); 723 return options; 724 } 725 726 727 728 /** 729 * Performs the actual processing for this tool. In this case, it gets a 730 * connection to the directory server and uses it to perform the requested 731 * searches. 732 * 733 * @return The result code for the processing that was performed. 734 */ 735 @Override() 736 public ResultCode doToolProcessing() 737 { 738 runningThread = Thread.currentThread(); 739 740 try 741 { 742 return doToolProcessingInternal(); 743 } 744 finally 745 { 746 runningThread = null; 747 } 748 } 749 750 751 752 /** 753 * Performs the actual processing for this tool. In this case, it gets a 754 * connection to the directory server and uses it to perform the requested 755 * searches. 756 * 757 * @return The result code for the processing that was performed. 758 */ 759 private ResultCode doToolProcessingInternal() 760 { 761 // If the sample rate file argument was specified, then generate the sample 762 // variable rate data file and return. 763 if (sampleRateFile.isPresent()) 764 { 765 try 766 { 767 RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue()); 768 return ResultCode.SUCCESS; 769 } 770 catch (final Exception e) 771 { 772 debugException(e); 773 err("An error occurred while trying to write sample variable data " + 774 "rate file '", sampleRateFile.getValue().getAbsolutePath(), 775 "': ", getExceptionMessage(e)); 776 return ResultCode.LOCAL_ERROR; 777 } 778 } 779 780 781 // Determine the random seed to use. 782 final Long seed; 783 if (randomSeed.isPresent()) 784 { 785 seed = Long.valueOf(randomSeed.getValue()); 786 } 787 else 788 { 789 seed = null; 790 } 791 792 // Create value patterns for the base DN, filter, and proxied authorization 793 // DN. 794 final ValuePattern dnPattern; 795 try 796 { 797 dnPattern = new ValuePattern(baseDN.getValue(), seed); 798 } 799 catch (final ParseException pe) 800 { 801 debugException(pe); 802 err("Unable to parse the base DN value pattern: ", pe.getMessage()); 803 return ResultCode.PARAM_ERROR; 804 } 805 806 final ValuePattern filterPattern; 807 try 808 { 809 filterPattern = new ValuePattern(filter.getValue(), seed); 810 } 811 catch (final ParseException pe) 812 { 813 debugException(pe); 814 err("Unable to parse the filter pattern: ", pe.getMessage()); 815 return ResultCode.PARAM_ERROR; 816 } 817 818 final ValuePattern authzIDPattern; 819 if (proxyAs.isPresent()) 820 { 821 try 822 { 823 authzIDPattern = new ValuePattern(proxyAs.getValue(), seed); 824 } 825 catch (final ParseException pe) 826 { 827 debugException(pe); 828 err("Unable to parse the proxied authorization pattern: ", 829 pe.getMessage()); 830 return ResultCode.PARAM_ERROR; 831 } 832 } 833 else 834 { 835 authzIDPattern = null; 836 } 837 838 839 // Get the attributes to return. 840 final String[] returnAttrs; 841 if (returnAttributes.isPresent()) 842 { 843 final List<String> attrList = returnAttributes.getValues(); 844 returnAttrs = new String[attrList.size()]; 845 attrList.toArray(returnAttrs); 846 } 847 else 848 { 849 returnAttrs = NO_STRINGS; 850 } 851 852 853 // Get the names of the attributes to modify. 854 final String[] modAttrs = new String[modifyAttributes.getValues().size()]; 855 modifyAttributes.getValues().toArray(modAttrs); 856 857 858 // Get the character set as a byte array. 859 final byte[] charSet = getBytes(characterSet.getValue()); 860 861 862 // If the --ratePerSecond option was specified, then limit the rate 863 // accordingly. 864 FixedRateBarrier fixedRateBarrier = null; 865 if (ratePerSecond.isPresent() || variableRateData.isPresent()) 866 { 867 // We might not have a rate per second if --variableRateData is specified. 868 // The rate typically doesn't matter except when we have warm-up 869 // intervals. In this case, we'll run at the max rate. 870 final int intervalSeconds = collectionInterval.getValue(); 871 final int ratePerInterval = 872 (ratePerSecond.getValue() == null) 873 ? Integer.MAX_VALUE 874 : ratePerSecond.getValue() * intervalSeconds; 875 fixedRateBarrier = 876 new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval); 877 } 878 879 880 // If --variableRateData was specified, then initialize a RateAdjustor. 881 RateAdjustor rateAdjustor = null; 882 if (variableRateData.isPresent()) 883 { 884 try 885 { 886 rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier, 887 ratePerSecond.getValue(), variableRateData.getValue()); 888 } 889 catch (final IOException e) 890 { 891 debugException(e); 892 err("Initializing the variable rates failed: " + e.getMessage()); 893 return ResultCode.PARAM_ERROR; 894 } 895 catch (final IllegalArgumentException e) 896 { 897 debugException(e); 898 err("Initializing the variable rates failed: " + e.getMessage()); 899 return ResultCode.PARAM_ERROR; 900 } 901 } 902 903 904 // Determine whether to include timestamps in the output and if so what 905 // format should be used for them. 906 final boolean includeTimestamp; 907 final String timeFormat; 908 if (timestampFormat.getValue().equalsIgnoreCase("with-date")) 909 { 910 includeTimestamp = true; 911 timeFormat = "dd/MM/yyyy HH:mm:ss"; 912 } 913 else if (timestampFormat.getValue().equalsIgnoreCase("without-date")) 914 { 915 includeTimestamp = true; 916 timeFormat = "HH:mm:ss"; 917 } 918 else 919 { 920 includeTimestamp = false; 921 timeFormat = null; 922 } 923 924 925 // Determine whether any warm-up intervals should be run. 926 final long totalIntervals; 927 final boolean warmUp; 928 int remainingWarmUpIntervals = warmUpIntervals.getValue(); 929 if (remainingWarmUpIntervals > 0) 930 { 931 warmUp = true; 932 totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals; 933 } 934 else 935 { 936 warmUp = true; 937 totalIntervals = 0L + numIntervals.getValue(); 938 } 939 940 941 // Create the table that will be used to format the output. 942 final OutputFormat outputFormat; 943 if (csvFormat.isPresent()) 944 { 945 outputFormat = OutputFormat.CSV; 946 } 947 else 948 { 949 outputFormat = OutputFormat.COLUMNS; 950 } 951 952 final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp, 953 timeFormat, outputFormat, " ", 954 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 955 "Searches/Sec"), 956 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 957 "Srch Dur ms"), 958 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 959 "Mods/Sec"), 960 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 961 "Mod Dur ms"), 962 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 963 "Errors/Sec"), 964 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", 965 "Searches/Sec"), 966 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", 967 "Srch Dur ms"), 968 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", 969 "Mods/Sec"), 970 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", 971 "Mod Dur ms")); 972 973 974 // Create values to use for statistics collection. 975 final AtomicLong searchCounter = new AtomicLong(0L); 976 final AtomicLong errorCounter = new AtomicLong(0L); 977 final AtomicLong modCounter = new AtomicLong(0L); 978 final AtomicLong modDurations = new AtomicLong(0L); 979 final AtomicLong searchDurations = new AtomicLong(0L); 980 final ResultCodeCounter rcCounter = new ResultCodeCounter(); 981 982 983 // Determine the length of each interval in milliseconds. 984 final long intervalMillis = 1000L * collectionInterval.getValue(); 985 986 987 // Create the threads to use for the searches. 988 final Random random = new Random(); 989 final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1); 990 final SearchAndModRateThread[] threads = 991 new SearchAndModRateThread[numThreads.getValue()]; 992 for (int i=0; i < threads.length; i++) 993 { 994 final LDAPConnection connection; 995 try 996 { 997 connection = getConnection(); 998 } 999 catch (final LDAPException le) 1000 { 1001 debugException(le); 1002 err("Unable to connect to the directory server: ", 1003 getExceptionMessage(le)); 1004 return le.getResultCode(); 1005 } 1006 1007 threads[i] = new SearchAndModRateThread(this, i, connection, dnPattern, 1008 scopeArg.getValue(), filterPattern, returnAttrs, modAttrs, 1009 valueLength.getValue(), charSet, authzIDPattern, 1010 iterationsBeforeReconnect.getValue(), random.nextLong(), barrier, 1011 searchCounter, modCounter, searchDurations, modDurations, 1012 errorCounter, rcCounter, fixedRateBarrier); 1013 threads[i].start(); 1014 } 1015 1016 1017 // Display the table header. 1018 for (final String headerLine : formatter.getHeaderLines(true)) 1019 { 1020 out(headerLine); 1021 } 1022 1023 1024 // Start the RateAdjustor before the threads so that the initial value is 1025 // in place before any load is generated unless we're doing a warm-up in 1026 // which case, we'll start it after the warm-up is complete. 1027 if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0)) 1028 { 1029 rateAdjustor.start(); 1030 } 1031 1032 1033 // Indicate that the threads can start running. 1034 try 1035 { 1036 barrier.await(); 1037 } 1038 catch (final Exception e) 1039 { 1040 debugException(e); 1041 } 1042 1043 long overallStartTime = System.nanoTime(); 1044 long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis; 1045 1046 1047 boolean setOverallStartTime = false; 1048 long lastSearchDuration = 0L; 1049 long lastModDuration = 0L; 1050 long lastNumErrors = 0L; 1051 long lastNumSearches = 0L; 1052 long lastNumMods = 0L; 1053 long lastEndTime = System.nanoTime(); 1054 for (long i=0; i < totalIntervals; i++) 1055 { 1056 if (rateAdjustor != null) 1057 { 1058 if (! rateAdjustor.isAlive()) 1059 { 1060 out("All of the rates in " + variableRateData.getValue().getName() + 1061 " have been completed."); 1062 break; 1063 } 1064 } 1065 1066 final long startTimeMillis = System.currentTimeMillis(); 1067 final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis; 1068 nextIntervalStartTime += intervalMillis; 1069 if (sleepTimeMillis > 0) 1070 { 1071 sleeper.sleep(sleepTimeMillis); 1072 } 1073 1074 if (stopRequested.get()) 1075 { 1076 break; 1077 } 1078 1079 final long endTime = System.nanoTime(); 1080 final long intervalDuration = endTime - lastEndTime; 1081 1082 final long numSearches; 1083 final long numMods; 1084 final long numErrors; 1085 final long totalSearchDuration; 1086 final long totalModDuration; 1087 if (warmUp && (remainingWarmUpIntervals > 0)) 1088 { 1089 numSearches = searchCounter.getAndSet(0L); 1090 numMods = modCounter.getAndSet(0L); 1091 numErrors = errorCounter.getAndSet(0L); 1092 totalSearchDuration = searchDurations.getAndSet(0L); 1093 totalModDuration = modDurations.getAndSet(0L); 1094 } 1095 else 1096 { 1097 numSearches = searchCounter.get(); 1098 numMods = modCounter.get(); 1099 numErrors = errorCounter.get(); 1100 totalSearchDuration = searchDurations.get(); 1101 totalModDuration = modDurations.get(); 1102 } 1103 1104 final long recentNumSearches = numSearches - lastNumSearches; 1105 final long recentNumMods = numMods - lastNumMods; 1106 final long recentNumErrors = numErrors - lastNumErrors; 1107 final long recentSearchDuration = 1108 totalSearchDuration - lastSearchDuration; 1109 final long recentModDuration = totalModDuration - lastModDuration; 1110 1111 final double numSeconds = intervalDuration / 1000000000.0d; 1112 final double recentSearchRate = recentNumSearches / numSeconds; 1113 final double recentModRate = recentNumMods / numSeconds; 1114 final double recentErrorRate = recentNumErrors / numSeconds; 1115 1116 final double recentAvgSearchDuration; 1117 if (recentNumSearches > 0L) 1118 { 1119 recentAvgSearchDuration = 1120 1.0d * recentSearchDuration / recentNumSearches / 1000000; 1121 } 1122 else 1123 { 1124 recentAvgSearchDuration = 0.0d; 1125 } 1126 1127 final double recentAvgModDuration; 1128 if (recentNumMods > 0L) 1129 { 1130 recentAvgModDuration = 1131 1.0d * recentModDuration / recentNumMods / 1000000; 1132 } 1133 else 1134 { 1135 recentAvgModDuration = 0.0d; 1136 } 1137 1138 if (warmUp && (remainingWarmUpIntervals > 0)) 1139 { 1140 out(formatter.formatRow(recentSearchRate, recentAvgSearchDuration, 1141 recentModRate, recentAvgModDuration, recentErrorRate, "warming up", 1142 "warming up", "warming up", "warming up")); 1143 1144 remainingWarmUpIntervals--; 1145 if (remainingWarmUpIntervals == 0) 1146 { 1147 out("Warm-up completed. Beginning overall statistics collection."); 1148 setOverallStartTime = true; 1149 if (rateAdjustor != null) 1150 { 1151 rateAdjustor.start(); 1152 } 1153 } 1154 } 1155 else 1156 { 1157 if (setOverallStartTime) 1158 { 1159 overallStartTime = lastEndTime; 1160 setOverallStartTime = false; 1161 } 1162 1163 final double numOverallSeconds = 1164 (endTime - overallStartTime) / 1000000000.0d; 1165 final double overallSearchRate = numSearches / numOverallSeconds; 1166 final double overallModRate = numMods / numOverallSeconds; 1167 1168 final double overallAvgSearchDuration; 1169 if (numSearches > 0L) 1170 { 1171 overallAvgSearchDuration = 1172 1.0d * totalSearchDuration / numSearches / 1000000; 1173 } 1174 else 1175 { 1176 overallAvgSearchDuration = 0.0d; 1177 } 1178 1179 final double overallAvgModDuration; 1180 if (numMods > 0L) 1181 { 1182 overallAvgModDuration = 1183 1.0d * totalModDuration / numMods / 1000000; 1184 } 1185 else 1186 { 1187 overallAvgModDuration = 0.0d; 1188 } 1189 1190 out(formatter.formatRow(recentSearchRate, recentAvgSearchDuration, 1191 recentModRate, recentAvgModDuration, recentErrorRate, 1192 overallSearchRate, overallAvgSearchDuration, overallModRate, 1193 overallAvgModDuration)); 1194 1195 lastNumSearches = numSearches; 1196 lastNumMods = numMods; 1197 lastNumErrors = numErrors; 1198 lastSearchDuration = totalSearchDuration; 1199 lastModDuration = totalModDuration; 1200 } 1201 1202 final List<ObjectPair<ResultCode,Long>> rcCounts = 1203 rcCounter.getCounts(true); 1204 if ((! suppressErrors.isPresent()) && (! rcCounts.isEmpty())) 1205 { 1206 err("\tError Results:"); 1207 for (final ObjectPair<ResultCode,Long> p : rcCounts) 1208 { 1209 err("\t", p.getFirst().getName(), ": ", p.getSecond()); 1210 } 1211 } 1212 1213 lastEndTime = endTime; 1214 } 1215 1216 1217 // Shut down the RateAdjustor if we have one. 1218 if (rateAdjustor != null) 1219 { 1220 rateAdjustor.shutDown(); 1221 } 1222 1223 // Stop all of the threads. 1224 ResultCode resultCode = ResultCode.SUCCESS; 1225 for (final SearchAndModRateThread t : threads) 1226 { 1227 final ResultCode r = t.stopRunning(); 1228 if (resultCode == ResultCode.SUCCESS) 1229 { 1230 resultCode = r; 1231 } 1232 } 1233 1234 return resultCode; 1235 } 1236 1237 1238 1239 /** 1240 * Requests that this tool stop running. This method will attempt to wait 1241 * for all threads to complete before returning control to the caller. 1242 */ 1243 public void stopRunning() 1244 { 1245 stopRequested.set(true); 1246 sleeper.wakeup(); 1247 1248 final Thread t = runningThread; 1249 if (t != null) 1250 { 1251 try 1252 { 1253 t.join(); 1254 } 1255 catch (final Exception e) 1256 { 1257 debugException(e); 1258 } 1259 } 1260 } 1261 1262 1263 1264 /** 1265 * {@inheritDoc} 1266 */ 1267 @Override() 1268 public LinkedHashMap<String[],String> getExampleUsages() 1269 { 1270 final LinkedHashMap<String[],String> examples = 1271 new LinkedHashMap<String[],String>(2); 1272 1273 String[] args = 1274 { 1275 "--hostname", "server.example.com", 1276 "--port", "389", 1277 "--bindDN", "uid=admin,dc=example,dc=com", 1278 "--bindPassword", "password", 1279 "--baseDN", "dc=example,dc=com", 1280 "--scope", "sub", 1281 "--filter", "(uid=user.[1-1000000])", 1282 "--attribute", "givenName", 1283 "--attribute", "sn", 1284 "--attribute", "mail", 1285 "--modifyAttribute", "description", 1286 "--valueLength", "10", 1287 "--characterSet", "abcdefghijklmnopqrstuvwxyz0123456789", 1288 "--numThreads", "10" 1289 }; 1290 String description = 1291 "Test search and modify performance by searching randomly across a " + 1292 "set of one million users located below 'dc=example,dc=com' with " + 1293 "ten concurrent threads. The entries returned to the client will " + 1294 "include the givenName, sn, and mail attributes, and the " + 1295 "description attribute of each entry returned will be replaced " + 1296 "with a string of ten randomly-selected alphanumeric characters."; 1297 examples.put(args, description); 1298 1299 args = new String[] 1300 { 1301 "--generateSampleRateFile", "variable-rate-data.txt" 1302 }; 1303 description = 1304 "Generate a sample variable rate definition file that may be used " + 1305 "in conjunction with the --variableRateData argument. The sample " + 1306 "file will include comments that describe the format for data to be " + 1307 "included in this file."; 1308 examples.put(args, description); 1309 1310 return examples; 1311 } 1312}