001/*
002 * Copyright 2008-2019 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright (C) 2008-2019 Ping Identity Corporation
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021package com.unboundid.ldap.sdk.examples;
022
023
024
025import java.io.IOException;
026import java.io.OutputStream;
027import java.io.Serializable;
028import java.text.ParseException;
029import java.util.ArrayList;
030import java.util.LinkedHashMap;
031import java.util.List;
032import java.util.Set;
033import java.util.concurrent.CyclicBarrier;
034import java.util.concurrent.atomic.AtomicBoolean;
035import java.util.concurrent.atomic.AtomicInteger;
036import java.util.concurrent.atomic.AtomicLong;
037
038import com.unboundid.ldap.sdk.Control;
039import com.unboundid.ldap.sdk.LDAPConnection;
040import com.unboundid.ldap.sdk.LDAPConnectionOptions;
041import com.unboundid.ldap.sdk.LDAPException;
042import com.unboundid.ldap.sdk.ResultCode;
043import com.unboundid.ldap.sdk.Version;
044import com.unboundid.ldap.sdk.controls.AssertionRequestControl;
045import com.unboundid.ldap.sdk.controls.PermissiveModifyRequestControl;
046import com.unboundid.ldap.sdk.controls.PostReadRequestControl;
047import com.unboundid.ldap.sdk.controls.PreReadRequestControl;
048import com.unboundid.util.ColumnFormatter;
049import com.unboundid.util.Debug;
050import com.unboundid.util.FixedRateBarrier;
051import com.unboundid.util.FormattableColumn;
052import com.unboundid.util.HorizontalAlignment;
053import com.unboundid.util.LDAPCommandLineTool;
054import com.unboundid.util.ObjectPair;
055import com.unboundid.util.OutputFormat;
056import com.unboundid.util.RateAdjustor;
057import com.unboundid.util.ResultCodeCounter;
058import com.unboundid.util.StaticUtils;
059import com.unboundid.util.ThreadSafety;
060import com.unboundid.util.ThreadSafetyLevel;
061import com.unboundid.util.ValuePattern;
062import com.unboundid.util.WakeableSleeper;
063import com.unboundid.util.args.ArgumentException;
064import com.unboundid.util.args.ArgumentParser;
065import com.unboundid.util.args.BooleanArgument;
066import com.unboundid.util.args.ControlArgument;
067import com.unboundid.util.args.FileArgument;
068import com.unboundid.util.args.FilterArgument;
069import com.unboundid.util.args.IntegerArgument;
070import com.unboundid.util.args.StringArgument;
071
072
073
074/**
075 * This class provides a tool that can be used to perform repeated modifications
076 * in an LDAP directory server using multiple threads.  It can help provide an
077 * estimate of the modify performance that a directory server is able to
078 * achieve.  The target entry DN may be a value pattern as described in the
079 * {@link ValuePattern} class.  This makes it possible to modify a range of
080 * entries rather than repeatedly updating the same entry.
081 * <BR><BR>
082 * Some of the APIs demonstrated by this example include:
083 * <UL>
084 *   <LI>Argument Parsing (from the {@code com.unboundid.util.args}
085 *       package)</LI>
086 *   <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util}
087 *       package)</LI>
088 *   <LI>LDAP Communication (from the {@code com.unboundid.ldap.sdk}
089 *       package)</LI>
090 *   <LI>Value Patterns (from the {@code com.unboundid.util} package)</LI>
091 * </UL>
092 * <BR><BR>
093 * All of the necessary information is provided using command line arguments.
094 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
095 * class, as well as the following additional arguments:
096 * <UL>
097 *   <LI>"-b {entryDN}" or "--targetDN {baseDN}" -- specifies the DN of the
098 *       entry to be modified.  This must be provided.  It may be a simple DN,
099 *       or it may be a value pattern to express a range of entry DNs.</LI>
100 *   <LI>"-A {name}" or "--attribute {name}" -- specifies the name of the
101 *       attribute to modify.  Multiple attributes may be modified by providing
102 *       multiple instances of this argument.  At least one attribute must be
103 *       provided.</LI>
104 *   <LI>"--valuePattern {pattern}" -- specifies the pattern to use to generate
105 *       the value to use for each modification.  If this argument is provided,
106 *       then neither the "--valueLength" nor "--characterSet" arguments may be
107 *       given.</LI>
108 *   <LI>"-l {num}" or "--valueLength {num}" -- specifies the length in bytes to
109 *       use for the values of the target attributes.  If this is not provided,
110 *       then a default length of 10 bytes will be used.</LI>
111 *   <LI>"-C {chars}" or "--characterSet {chars}" -- specifies the set of
112 *       characters that will be used to generate the values to use for the
113 *       target attributes.  It should only include ASCII characters.  Values
114 *       will be generated from randomly-selected characters from this set.  If
115 *       this is not provided, then a default set of lowercase alphabetic
116 *       characters will be used.</LI>
117 *   <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of
118 *       concurrent threads to use when performing the modifications.  If this
119 *       is not provided, then a default of one thread will be used.</LI>
120 *   <LI>"-i {sec}" or "--intervalDuration {sec}" -- specifies the length of
121 *       time in seconds between lines out output.  If this is not provided,
122 *       then a default interval duration of five seconds will be used.</LI>
123 *   <LI>"-I {num}" or "--numIntervals {num}" -- specifies the maximum number of
124 *       intervals for which to run.  If this is not provided, then it will
125 *       run forever.</LI>
126 *   <LI>"--iterationsBeforeReconnect {num}" -- specifies the number of modify
127 *       iterations that should be performed on a connection before that
128 *       connection is closed and replaced with a newly-established (and
129 *       authenticated, if appropriate) connection.</LI>
130 *   <LI>"-r {modifies-per-second}" or "--ratePerSecond {modifies-per-second}"
131 *       -- specifies the target number of modifies to perform per second.  It
132 *       is still necessary to specify a sufficient number of threads for
133 *       achieving this rate.  If this option is not provided, then the tool
134 *       will run at the maximum rate for the specified number of threads.</LI>
135 *   <LI>"--variableRateData {path}" -- specifies the path to a file containing
136 *       information needed to allow the tool to vary the target rate over time.
137 *       If this option is not provided, then the tool will either use a fixed
138 *       target rate as specified by the "--ratePerSecond" argument, or it will
139 *       run at the maximum rate.</LI>
140 *   <LI>"--generateSampleRateFile {path}" -- specifies the path to a file to
141 *       which sample data will be written illustrating and describing the
142 *       format of the file expected to be used in conjunction with the
143 *       "--variableRateData" argument.</LI>
144 *   <LI>"--warmUpIntervals {num}" -- specifies the number of intervals to
145 *       complete before beginning overall statistics collection.</LI>
146 *   <LI>"--timestampFormat {format}" -- specifies the format to use for
147 *       timestamps included before each output line.  The format may be one of
148 *       "none" (for no timestamps), "with-date" (to include both the date and
149 *       the time), or "without-date" (to include only time time).</LI>
150 *   <LI>"-Y {authzID}" or "--proxyAs {authzID}" -- Use the proxied
151 *       authorization v2 control to request that the operation be processed
152 *       using an alternate authorization identity.  In this case, the bind DN
153 *       should be that of a user that has permission to use this control.  The
154 *       authorization identity may be a value pattern.</LI>
155 *   <LI>"--suppressErrorResultCodes" -- Indicates that information about the
156 *       result codes for failed operations should not be displayed.</LI>
157 *   <LI>"-c" or "--csv" -- Generate output in CSV format rather than a
158 *       display-friendly format.</LI>
159 * </UL>
160 */
161@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
162public final class ModRate
163       extends LDAPCommandLineTool
164       implements Serializable
165{
166  /**
167   * The serial version UID for this serializable class.
168   */
169  private static final long serialVersionUID = 2709717414202815822L;
170
171
172
173  // Indicates whether a request has been made to stop running.
174  private final AtomicBoolean stopRequested;
175
176  // The number of modrate threads that are currently running.
177  private final AtomicInteger runningThreads;
178
179  // The argument used to indicate whether to generate output in CSV format.
180  private BooleanArgument csvFormat;
181
182  // Indicates that the tool should use the increment modification type instead
183  // of replace.
184  private BooleanArgument increment;
185
186  // Indicates that modify requests should include the permissive modify request
187  // control.
188  private BooleanArgument permissiveModify;
189
190  // The argument used to indicate whether to suppress information about error
191  // result codes.
192  private BooleanArgument suppressErrorsArgument;
193
194  // The argument used to indicate that a generic control should be included in
195  // the request.
196  private ControlArgument control;
197
198  // The argument used to specify a variable rate file.
199  private FileArgument sampleRateFile;
200
201  // The argument used to specify a variable rate file.
202  private FileArgument variableRateData;
203
204  // Indicates that modify requests should include the assertion request control
205  // with the specified filter.
206  private FilterArgument assertionFilter;
207
208  // The argument used to specify the collection interval.
209  private IntegerArgument collectionInterval;
210
211  // The increment amount to use when performing an increment instead of a
212  // replace.
213  private IntegerArgument incrementAmount;
214
215  // The argument used to specify the number of modify iterations on a
216  // connection before it is closed and re-established.
217  private IntegerArgument iterationsBeforeReconnect;
218
219  // The argument used to specify the number of intervals.
220  private IntegerArgument numIntervals;
221
222  // The argument used to specify the number of threads.
223  private IntegerArgument numThreads;
224
225  // The argument used to specify the seed to use for the random number
226  // generator.
227  private IntegerArgument randomSeed;
228
229  // The target rate of modifies per second.
230  private IntegerArgument ratePerSecond;
231
232  // The number of values to include in the replace modification.
233  private IntegerArgument valueCount;
234
235  // The argument used to specify the length of the values to generate.
236  private IntegerArgument valueLength;
237
238  // The number of warm-up intervals to perform.
239  private IntegerArgument warmUpIntervals;
240
241  // The argument used to specify the name of the attribute to modify.
242  private StringArgument attribute;
243
244  // The argument used to specify the set of characters to use when generating
245  // values.
246  private StringArgument characterSet;
247
248  // The argument used to specify the DNs of the entries to modify.
249  private StringArgument entryDN;
250
251  // Indicates that modify requests should include the post-read request control
252  // to request the specified attribute.
253  private StringArgument postReadAttribute;
254
255  // Indicates that modify requests should include the pre-read request control
256  // to request the specified attribute.
257  private StringArgument preReadAttribute;
258
259  // The argument used to specify the proxied authorization identity.
260  private StringArgument proxyAs;
261
262  // The argument used to specify the timestamp format.
263  private StringArgument timestampFormat;
264
265  // The argument used to specify the pattern to use to generate values.
266  private StringArgument valuePattern;
267
268  // A wakeable sleeper that will be used to sleep between reporting intervals.
269  private final WakeableSleeper sleeper;
270
271
272
273  /**
274   * Parse the provided command line arguments and make the appropriate set of
275   * changes.
276   *
277   * @param  args  The command line arguments provided to this program.
278   */
279  public static void main(final String[] args)
280  {
281    final ResultCode resultCode = main(args, System.out, System.err);
282    if (resultCode != ResultCode.SUCCESS)
283    {
284      System.exit(resultCode.intValue());
285    }
286  }
287
288
289
290  /**
291   * Parse the provided command line arguments and make the appropriate set of
292   * changes.
293   *
294   * @param  args       The command line arguments provided to this program.
295   * @param  outStream  The output stream to which standard out should be
296   *                    written.  It may be {@code null} if output should be
297   *                    suppressed.
298   * @param  errStream  The output stream to which standard error should be
299   *                    written.  It may be {@code null} if error messages
300   *                    should be suppressed.
301   *
302   * @return  A result code indicating whether the processing was successful.
303   */
304  public static ResultCode main(final String[] args,
305                                final OutputStream outStream,
306                                final OutputStream errStream)
307  {
308    final ModRate modRate = new ModRate(outStream, errStream);
309    return modRate.runTool(args);
310  }
311
312
313
314  /**
315   * Creates a new instance of this tool.
316   *
317   * @param  outStream  The output stream to which standard out should be
318   *                    written.  It may be {@code null} if output should be
319   *                    suppressed.
320   * @param  errStream  The output stream to which standard error should be
321   *                    written.  It may be {@code null} if error messages
322   *                    should be suppressed.
323   */
324  public ModRate(final OutputStream outStream, final OutputStream errStream)
325  {
326    super(outStream, errStream);
327
328    stopRequested = new AtomicBoolean(false);
329    runningThreads = new AtomicInteger(0);
330    sleeper = new WakeableSleeper();
331  }
332
333
334
335  /**
336   * Retrieves the name for this tool.
337   *
338   * @return  The name for this tool.
339   */
340  @Override()
341  public String getToolName()
342  {
343    return "modrate";
344  }
345
346
347
348  /**
349   * Retrieves the description for this tool.
350   *
351   * @return  The description for this tool.
352   */
353  @Override()
354  public String getToolDescription()
355  {
356    return "Perform repeated modifications against " +
357           "an LDAP directory server.";
358  }
359
360
361
362  /**
363   * Retrieves the version string for this tool.
364   *
365   * @return  The version string for this tool.
366   */
367  @Override()
368  public String getToolVersion()
369  {
370    return Version.NUMERIC_VERSION_STRING;
371  }
372
373
374
375  /**
376   * Indicates whether this tool should provide support for an interactive mode,
377   * in which the tool offers a mode in which the arguments can be provided in
378   * a text-driven menu rather than requiring them to be given on the command
379   * line.  If interactive mode is supported, it may be invoked using the
380   * "--interactive" argument.  Alternately, if interactive mode is supported
381   * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
382   * interactive mode may be invoked by simply launching the tool without any
383   * arguments.
384   *
385   * @return  {@code true} if this tool supports interactive mode, or
386   *          {@code false} if not.
387   */
388  @Override()
389  public boolean supportsInteractiveMode()
390  {
391    return true;
392  }
393
394
395
396  /**
397   * Indicates whether this tool defaults to launching in interactive mode if
398   * the tool is invoked without any command-line arguments.  This will only be
399   * used if {@link #supportsInteractiveMode()} returns {@code true}.
400   *
401   * @return  {@code true} if this tool defaults to using interactive mode if
402   *          launched without any command-line arguments, or {@code false} if
403   *          not.
404   */
405  @Override()
406  public boolean defaultsToInteractiveMode()
407  {
408    return true;
409  }
410
411
412
413  /**
414   * Indicates whether this tool should provide arguments for redirecting output
415   * to a file.  If this method returns {@code true}, then the tool will offer
416   * an "--outputFile" argument that will specify the path to a file to which
417   * all standard output and standard error content will be written, and it will
418   * also offer a "--teeToStandardOut" argument that can only be used if the
419   * "--outputFile" argument is present and will cause all output to be written
420   * to both the specified output file and to standard output.
421   *
422   * @return  {@code true} if this tool should provide arguments for redirecting
423   *          output to a file, or {@code false} if not.
424   */
425  @Override()
426  protected boolean supportsOutputFile()
427  {
428    return true;
429  }
430
431
432
433  /**
434   * Indicates whether this tool should default to interactively prompting for
435   * the bind password if a password is required but no argument was provided
436   * to indicate how to get the password.
437   *
438   * @return  {@code true} if this tool should default to interactively
439   *          prompting for the bind password, or {@code false} if not.
440   */
441  @Override()
442  protected boolean defaultToPromptForBindPassword()
443  {
444    return true;
445  }
446
447
448
449  /**
450   * Indicates whether this tool supports the use of a properties file for
451   * specifying default values for arguments that aren't specified on the
452   * command line.
453   *
454   * @return  {@code true} if this tool supports the use of a properties file
455   *          for specifying default values for arguments that aren't specified
456   *          on the command line, or {@code false} if not.
457   */
458  @Override()
459  public boolean supportsPropertiesFile()
460  {
461    return true;
462  }
463
464
465
466  /**
467   * Indicates whether the LDAP-specific arguments should include alternate
468   * versions of all long identifiers that consist of multiple words so that
469   * they are available in both camelCase and dash-separated versions.
470   *
471   * @return  {@code true} if this tool should provide multiple versions of
472   *          long identifiers for LDAP-specific arguments, or {@code false} if
473   *          not.
474   */
475  @Override()
476  protected boolean includeAlternateLongIdentifiers()
477  {
478    return true;
479  }
480
481
482
483  /**
484   * {@inheritDoc}
485   */
486  @Override()
487  protected boolean logToolInvocationByDefault()
488  {
489    return true;
490  }
491
492
493
494  /**
495   * Adds the arguments used by this program that aren't already provided by the
496   * generic {@code LDAPCommandLineTool} framework.
497   *
498   * @param  parser  The argument parser to which the arguments should be added.
499   *
500   * @throws  ArgumentException  If a problem occurs while adding the arguments.
501   */
502  @Override()
503  public void addNonLDAPArguments(final ArgumentParser parser)
504         throws ArgumentException
505  {
506    String description = "The DN of the entry to modify.  It may be a simple " +
507         "DN or a value pattern to specify a range of DN (e.g., " +
508         "\"uid=user.[1-1000],ou=People,dc=example,dc=com\").  See " +
509         ValuePattern.PUBLIC_JAVADOC_URL + " for complete details about the " +
510         "value pattern syntax.  This must be provided.";
511    entryDN = new StringArgument('b', "entryDN", true, 1, "{dn}", description);
512    entryDN.setArgumentGroupName("Modification Arguments");
513    entryDN.addLongIdentifier("entry-dn", true);
514    parser.addArgument(entryDN);
515
516
517    description = "The name of the attribute to modify.  Multiple attributes " +
518                  "may be specified by providing this argument multiple " +
519                  "times.  At least one attribute must be specified.";
520    attribute = new StringArgument('A', "attribute", true, 0, "{name}",
521                                   description);
522    attribute.setArgumentGroupName("Modification Arguments");
523    parser.addArgument(attribute);
524
525
526    description = "The pattern to use to generate values for the replace " +
527                  "modifications.  If this is provided, then neither the " +
528                  "--valueLength argument nor the --characterSet arguments " +
529                  "may be provided.";
530    valuePattern = new StringArgument(null, "valuePattern", false, 1,
531                                     "{pattern}", description);
532    valuePattern.setArgumentGroupName("Modification Arguments");
533    valuePattern.addLongIdentifier("value-pattern", true);
534    parser.addArgument(valuePattern);
535
536
537    description = "The length in bytes to use when generating values for the " +
538                  "replace modifications.  If this is not provided, then a " +
539                  "default length of ten bytes will be used.";
540    valueLength = new IntegerArgument('l', "valueLength", false, 1, "{num}",
541                                      description, 1, Integer.MAX_VALUE);
542    valueLength.setArgumentGroupName("Modification Arguments");
543    valueLength.addLongIdentifier("value-length", true);
544    parser.addArgument(valueLength);
545
546
547    description = "The number of values to include in replace " +
548                  "modifications.  If this is not provided, then a default " +
549                  "of one value will be used.";
550    valueCount = new IntegerArgument(null, "valueCount", false, 1, "{num}",
551                                     description, 0, Integer.MAX_VALUE, 1);
552    valueCount.setArgumentGroupName("Modification Arguments");
553    valueCount.addLongIdentifier("value-count", true);
554    parser.addArgument(valueCount);
555
556
557    description = "Indicates that the tool should use the increment " +
558                  "modification type rather than the replace modification " +
559                  "type.";
560    increment = new BooleanArgument(null, "increment", 1, description);
561    increment.setArgumentGroupName("Modification Arguments");
562    parser.addArgument(increment);
563
564
565    description = "The amount by which to increment values when using the " +
566                  "increment modification type.  The amount may be negative " +
567                  "if values should be decremented rather than incremented.  " +
568                  "If this is not provided, then a default increment amount " +
569                  "of one will be used.";
570    incrementAmount = new IntegerArgument(null, "incrementAmount", false, 1,
571                                          null, description, Integer.MIN_VALUE,
572                                          Integer.MAX_VALUE, 1);
573    incrementAmount.setArgumentGroupName("Modification Arguments");
574    incrementAmount.addLongIdentifier("increment-amount", true);
575    parser.addArgument(incrementAmount);
576
577
578    description = "The set of characters to use to generate the values for " +
579                  "the modifications.  It should only include ASCII " +
580                  "characters.  If this is not provided, then a default set " +
581                  "of lowercase alphabetic characters will be used.";
582    characterSet = new StringArgument('C', "characterSet", false, 1, "{chars}",
583                                      description);
584    characterSet.setArgumentGroupName("Modification Arguments");
585    characterSet.addLongIdentifier("character-set", true);
586    parser.addArgument(characterSet);
587
588
589    description = "Indicates that modify requests should include the " +
590                  "assertion request control with the specified filter.";
591    assertionFilter = new FilterArgument(null, "assertionFilter", false, 1,
592                                         "{filter}", description);
593    assertionFilter.setArgumentGroupName("Request Control Arguments");
594    assertionFilter.addLongIdentifier("assertion-filter", true);
595    parser.addArgument(assertionFilter);
596
597
598    description = "Indicates that modify requests should include the " +
599                  "permissive modify request control.";
600    permissiveModify = new BooleanArgument(null, "permissiveModify", 1,
601                                           description);
602    permissiveModify.setArgumentGroupName("Request Control Arguments");
603    permissiveModify.addLongIdentifier("permissive-modify", true);
604    parser.addArgument(permissiveModify);
605
606
607    description = "Indicates that modify requests should include the " +
608                  "pre-read request control with the specified requested " +
609                  "attribute.  This argument may be provided multiple times " +
610                  "to indicate that multiple requested attributes should be " +
611                  "included in the pre-read request control.";
612    preReadAttribute = new StringArgument(null, "preReadAttribute", false, 0,
613                                          "{attribute}", description);
614    preReadAttribute.setArgumentGroupName("Request Control Arguments");
615    preReadAttribute.addLongIdentifier("pre-read-attribute", true);
616    parser.addArgument(preReadAttribute);
617
618
619    description = "Indicates that modify requests should include the " +
620                  "post-read request control with the specified requested " +
621                  "attribute.  This argument may be provided multiple times " +
622                  "to indicate that multiple requested attributes should be " +
623                  "included in the post-read request control.";
624    postReadAttribute = new StringArgument(null, "postReadAttribute", false, 0,
625                                           "{attribute}", description);
626    postReadAttribute.setArgumentGroupName("Request Control Arguments");
627    postReadAttribute.addLongIdentifier("post-read-attribute", true);
628    parser.addArgument(postReadAttribute);
629
630
631    description = "Indicates that the proxied authorization control (as " +
632                  "defined in RFC 4370) should be used to request that " +
633                  "operations be processed using an alternate authorization " +
634                  "identity.  This may be a simple authorization ID or it " +
635                  "may be a value pattern to specify a range of " +
636                  "identities.  See " + ValuePattern.PUBLIC_JAVADOC_URL +
637                  " for complete details about the value pattern syntax.";
638    proxyAs = new StringArgument('Y', "proxyAs", false, 1, "{authzID}",
639                                 description);
640    proxyAs.setArgumentGroupName("Request Control Arguments");
641    proxyAs.addLongIdentifier("proxy-as", true);
642    parser.addArgument(proxyAs);
643
644
645    description = "Indicates that modify requests should include the " +
646                  "specified request control.  This may be provided multiple " +
647                  "times to include multiple request controls.";
648    control = new ControlArgument('J', "control", false, 0, null, description);
649    control.setArgumentGroupName("Request Control Arguments");
650    parser.addArgument(control);
651
652
653    description = "The number of threads to use to perform the " +
654                  "modifications.  If this is not provided, a single thread " +
655                  "will be used.";
656    numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}",
657                                     description, 1, Integer.MAX_VALUE, 1);
658    numThreads.setArgumentGroupName("Rate Management Arguments");
659    numThreads.addLongIdentifier("num-threads", true);
660    parser.addArgument(numThreads);
661
662
663    description = "The length of time in seconds between output lines.  If " +
664                  "this is not provided, then a default interval of five " +
665                  "seconds will be used.";
666    collectionInterval = new IntegerArgument('i', "intervalDuration", true, 1,
667                                             "{num}", description, 1,
668                                             Integer.MAX_VALUE, 5);
669    collectionInterval.setArgumentGroupName("Rate Management Arguments");
670    collectionInterval.addLongIdentifier("interval-duration", true);
671    parser.addArgument(collectionInterval);
672
673
674    description = "The maximum number of intervals for which to run.  If " +
675                  "this is not provided, then the tool will run until it is " +
676                  "interrupted.";
677    numIntervals = new IntegerArgument('I', "numIntervals", true, 1, "{num}",
678                                       description, 1, Integer.MAX_VALUE,
679                                       Integer.MAX_VALUE);
680    numIntervals.setArgumentGroupName("Rate Management Arguments");
681    numIntervals.addLongIdentifier("num-intervals", true);
682    parser.addArgument(numIntervals);
683
684    description = "The number of modify iterations that should be processed " +
685                  "on a connection before that connection is closed and " +
686                  "replaced with a newly-established (and authenticated, if " +
687                  "appropriate) connection.  If this is not provided, then " +
688                  "connections will not be periodically closed and " +
689                  "re-established.";
690    iterationsBeforeReconnect = new IntegerArgument(null,
691         "iterationsBeforeReconnect", false, 1, "{num}", description, 0);
692    iterationsBeforeReconnect.setArgumentGroupName("Rate Management Arguments");
693    iterationsBeforeReconnect.addLongIdentifier("iterations-before-reconnect",
694         true);
695    parser.addArgument(iterationsBeforeReconnect);
696
697    description = "The target number of modifies to perform per second.  It " +
698                  "is still necessary to specify a sufficient number of " +
699                  "threads for achieving this rate.  If neither this option " +
700                  "nor --variableRateData is provided, then the tool will " +
701                  "run at the maximum rate for the specified number of " +
702                  "threads.";
703    ratePerSecond = new IntegerArgument('r', "ratePerSecond", false, 1,
704                                        "{modifies-per-second}", description,
705                                        1, Integer.MAX_VALUE);
706    ratePerSecond.setArgumentGroupName("Rate Management Arguments");
707    ratePerSecond.addLongIdentifier("rate-per-second", true);
708    parser.addArgument(ratePerSecond);
709
710    final String variableRateDataArgName = "variableRateData";
711    final String generateSampleRateFileArgName = "generateSampleRateFile";
712    description = RateAdjustor.getVariableRateDataArgumentDescription(
713         generateSampleRateFileArgName);
714    variableRateData = new FileArgument(null, variableRateDataArgName, false, 1,
715                                        "{path}", description, true, true, true,
716                                        false);
717    variableRateData.setArgumentGroupName("Rate Management Arguments");
718    variableRateData.addLongIdentifier("variable-rate-data", true);
719    parser.addArgument(variableRateData);
720
721    description = RateAdjustor.getGenerateSampleVariableRateFileDescription(
722         variableRateDataArgName);
723    sampleRateFile = new FileArgument(null, generateSampleRateFileArgName,
724                                      false, 1, "{path}", description, false,
725                                      true, true, false);
726    sampleRateFile.setArgumentGroupName("Rate Management Arguments");
727    sampleRateFile.addLongIdentifier("generate-sample-rate-file", true);
728    sampleRateFile.setUsageArgument(true);
729    parser.addArgument(sampleRateFile);
730    parser.addExclusiveArgumentSet(variableRateData, sampleRateFile);
731
732    description = "The number of intervals to complete before beginning " +
733                  "overall statistics collection.  Specifying a nonzero " +
734                  "number of warm-up intervals gives the client and server " +
735                  "a chance to warm up without skewing performance results.";
736    warmUpIntervals = new IntegerArgument(null, "warmUpIntervals", true, 1,
737         "{num}", description, 0, Integer.MAX_VALUE, 0);
738    warmUpIntervals.setArgumentGroupName("Rate Management Arguments");
739    warmUpIntervals.addLongIdentifier("warm-up-intervals", true);
740    parser.addArgument(warmUpIntervals);
741
742    description = "Indicates the format to use for timestamps included in " +
743                  "the output.  A value of 'none' indicates that no " +
744                  "timestamps should be included.  A value of 'with-date' " +
745                  "indicates that both the date and the time should be " +
746                  "included.  A value of 'without-date' indicates that only " +
747                  "the time should be included.";
748    final Set<String> allowedFormats =
749         StaticUtils.setOf("none", "with-date", "without-date");
750    timestampFormat = new StringArgument(null, "timestampFormat", true, 1,
751         "{format}", description, allowedFormats, "none");
752    timestampFormat.addLongIdentifier("timestamp-format", true);
753    parser.addArgument(timestampFormat);
754
755    description = "Indicates that information about the result codes for " +
756                  "failed operations should not be displayed.";
757    suppressErrorsArgument = new BooleanArgument(null,
758         "suppressErrorResultCodes", 1, description);
759    suppressErrorsArgument.addLongIdentifier("suppress-error-result-codes",
760         true);
761    parser.addArgument(suppressErrorsArgument);
762
763    description = "Generate output in CSV format rather than a " +
764                  "display-friendly format";
765    csvFormat = new BooleanArgument('c', "csv", 1, description);
766    parser.addArgument(csvFormat);
767
768    description = "Specifies the seed to use for the random number generator.";
769    randomSeed = new IntegerArgument('R', "randomSeed", false, 1, "{value}",
770         description);
771    randomSeed.addLongIdentifier("random-seed", true);
772    parser.addArgument(randomSeed);
773
774
775    // The incrementAmount argument can only be used if the increment argument
776    // is provided.
777    parser.addDependentArgumentSet(incrementAmount, increment);
778
779
780    // None of the valueLength, valueCount, characterSet, or valuePattern
781    // arguments can be used if the increment argument is provided.
782    parser.addExclusiveArgumentSet(increment, valueLength);
783    parser.addExclusiveArgumentSet(increment, valueCount);
784    parser.addExclusiveArgumentSet(increment, characterSet);
785    parser.addExclusiveArgumentSet(increment, valuePattern);
786
787
788    // The valuePattern argument cannot be used with either the valueLength or
789    // characterSet arguments.
790    parser.addExclusiveArgumentSet(valuePattern, valueLength);
791    parser.addExclusiveArgumentSet(valuePattern, characterSet);
792  }
793
794
795
796  /**
797   * Indicates whether this tool supports creating connections to multiple
798   * servers.  If it is to support multiple servers, then the "--hostname" and
799   * "--port" arguments will be allowed to be provided multiple times, and
800   * will be required to be provided the same number of times.  The same type of
801   * communication security and bind credentials will be used for all servers.
802   *
803   * @return  {@code true} if this tool supports creating connections to
804   *          multiple servers, or {@code false} if not.
805   */
806  @Override()
807  protected boolean supportsMultipleServers()
808  {
809    return true;
810  }
811
812
813
814  /**
815   * Retrieves the connection options that should be used for connections
816   * created for use with this tool.
817   *
818   * @return  The connection options that should be used for connections created
819   *          for use with this tool.
820   */
821  @Override()
822  public LDAPConnectionOptions getConnectionOptions()
823  {
824    final LDAPConnectionOptions options = new LDAPConnectionOptions();
825    options.setUseSynchronousMode(true);
826    return options;
827  }
828
829
830
831  /**
832   * Performs the actual processing for this tool.  In this case, it gets a
833   * connection to the directory server and uses it to perform the requested
834   * modifications.
835   *
836   * @return  The result code for the processing that was performed.
837   */
838  @Override()
839  public ResultCode doToolProcessing()
840  {
841    // If the sample rate file argument was specified, then generate the sample
842    // variable rate data file and return.
843    if (sampleRateFile.isPresent())
844    {
845      try
846      {
847        RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue());
848        return ResultCode.SUCCESS;
849      }
850      catch (final Exception e)
851      {
852        Debug.debugException(e);
853        err("An error occurred while trying to write sample variable data " +
854             "rate file '", sampleRateFile.getValue().getAbsolutePath(),
855             "':  ", StaticUtils.getExceptionMessage(e));
856        return ResultCode.LOCAL_ERROR;
857      }
858    }
859
860
861    // Determine the random seed to use.
862    final Long seed;
863    if (randomSeed.isPresent())
864    {
865      seed = Long.valueOf(randomSeed.getValue());
866    }
867    else
868    {
869      seed = null;
870    }
871
872    // Create the value patterns for the target entry DN and proxied
873    // authorization identities.
874    final ValuePattern dnPattern;
875    try
876    {
877      dnPattern = new ValuePattern(entryDN.getValue(), seed);
878    }
879    catch (final ParseException pe)
880    {
881      Debug.debugException(pe);
882      err("Unable to parse the entry DN value pattern:  ", pe.getMessage());
883      return ResultCode.PARAM_ERROR;
884    }
885
886    final ValuePattern authzIDPattern;
887    if (proxyAs.isPresent())
888    {
889      try
890      {
891        authzIDPattern = new ValuePattern(proxyAs.getValue(), seed);
892      }
893      catch (final ParseException pe)
894      {
895        Debug.debugException(pe);
896        err("Unable to parse the proxied authorization pattern:  ",
897            pe.getMessage());
898        return ResultCode.PARAM_ERROR;
899      }
900    }
901    else
902    {
903      authzIDPattern = null;
904    }
905
906
907    // Get the set of controls to include in modify requests.
908    final ArrayList<Control> controlList = new ArrayList<>(5);
909    if (assertionFilter.isPresent())
910    {
911      controlList.add(new AssertionRequestControl(assertionFilter.getValue()));
912    }
913
914    if (permissiveModify.isPresent())
915    {
916      controlList.add(new PermissiveModifyRequestControl());
917    }
918
919    if (preReadAttribute.isPresent())
920    {
921      final List<String> attrList = preReadAttribute.getValues();
922      final String[] attrArray = new String[attrList.size()];
923      attrList.toArray(attrArray);
924      controlList.add(new PreReadRequestControl(attrArray));
925    }
926
927    if (postReadAttribute.isPresent())
928    {
929      final List<String> attrList = postReadAttribute.getValues();
930      final String[] attrArray = new String[attrList.size()];
931      attrList.toArray(attrArray);
932      controlList.add(new PostReadRequestControl(attrArray));
933    }
934
935    if (control.isPresent())
936    {
937      controlList.addAll(control.getValues());
938    }
939
940    final Control[] controlArray = new Control[controlList.size()];
941    controlList.toArray(controlArray);
942
943
944    // Get the names of the attributes to modify.
945    final String[] attrs = new String[attribute.getValues().size()];
946    attribute.getValues().toArray(attrs);
947
948
949    // If the --ratePerSecond option was specified, then limit the rate
950    // accordingly.
951    FixedRateBarrier fixedRateBarrier = null;
952    if (ratePerSecond.isPresent() || variableRateData.isPresent())
953    {
954      // We might not have a rate per second if --variableRateData is specified.
955      // The rate typically doesn't matter except when we have warm-up
956      // intervals.  In this case, we'll run at the max rate.
957      final int intervalSeconds = collectionInterval.getValue();
958      final int ratePerInterval =
959           (ratePerSecond.getValue() == null)
960           ? Integer.MAX_VALUE
961           : ratePerSecond.getValue() * intervalSeconds;
962      fixedRateBarrier =
963           new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval);
964    }
965
966
967    // If --variableRateData was specified, then initialize a RateAdjustor.
968    RateAdjustor rateAdjustor = null;
969    if (variableRateData.isPresent())
970    {
971      try
972      {
973        rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier,
974             ratePerSecond.getValue(), variableRateData.getValue());
975      }
976      catch (final IOException | IllegalArgumentException e)
977      {
978        Debug.debugException(e);
979        err("Initializing the variable rates failed: " + e.getMessage());
980        return ResultCode.PARAM_ERROR;
981      }
982    }
983
984
985    // Determine whether to include timestamps in the output and if so what
986    // format should be used for them.
987    final boolean includeTimestamp;
988    final String timeFormat;
989    if (timestampFormat.getValue().equalsIgnoreCase("with-date"))
990    {
991      includeTimestamp = true;
992      timeFormat       = "dd/MM/yyyy HH:mm:ss";
993    }
994    else if (timestampFormat.getValue().equalsIgnoreCase("without-date"))
995    {
996      includeTimestamp = true;
997      timeFormat       = "HH:mm:ss";
998    }
999    else
1000    {
1001      includeTimestamp = false;
1002      timeFormat       = null;
1003    }
1004
1005
1006    // Determine whether any warm-up intervals should be run.
1007    final long totalIntervals;
1008    final boolean warmUp;
1009    int remainingWarmUpIntervals = warmUpIntervals.getValue();
1010    if (remainingWarmUpIntervals > 0)
1011    {
1012      warmUp = true;
1013      totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals;
1014    }
1015    else
1016    {
1017      warmUp = true;
1018      totalIntervals = 0L + numIntervals.getValue();
1019    }
1020
1021
1022    // Create the table that will be used to format the output.
1023    final OutputFormat outputFormat;
1024    if (csvFormat.isPresent())
1025    {
1026      outputFormat = OutputFormat.CSV;
1027    }
1028    else
1029    {
1030      outputFormat = OutputFormat.COLUMNS;
1031    }
1032
1033    final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp,
1034         timeFormat, outputFormat, " ",
1035         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
1036                  "Mods/Sec"),
1037         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
1038                  "Avg Dur ms"),
1039         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
1040                  "Errors/Sec"),
1041         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
1042                  "Mods/Sec"),
1043         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
1044                  "Avg Dur ms"));
1045
1046
1047    // Create values to use for statistics collection.
1048    final AtomicLong        modCounter   = new AtomicLong(0L);
1049    final AtomicLong        errorCounter = new AtomicLong(0L);
1050    final AtomicLong        modDurations = new AtomicLong(0L);
1051    final ResultCodeCounter rcCounter    = new ResultCodeCounter();
1052
1053
1054    // Determine the length of each interval in milliseconds.
1055    final long intervalMillis = 1000L * collectionInterval.getValue();
1056
1057
1058    // Create the threads to use for the modifications.
1059    final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1);
1060    final ModRateThread[] threads = new ModRateThread[numThreads.getValue()];
1061    for (int i=0; i < threads.length; i++)
1062    {
1063      final LDAPConnection connection;
1064      try
1065      {
1066        connection = getConnection();
1067      }
1068      catch (final LDAPException le)
1069      {
1070        Debug.debugException(le);
1071        err("Unable to connect to the directory server:  ",
1072            StaticUtils.getExceptionMessage(le));
1073        return le.getResultCode();
1074      }
1075
1076      final String valuePatternString;
1077      if (valuePattern.isPresent())
1078      {
1079        valuePatternString = valuePattern.getValue();
1080      }
1081      else
1082      {
1083        final int length;
1084        if (valueLength.isPresent())
1085        {
1086          length = valueLength.getValue();
1087        }
1088        else
1089        {
1090          length = 10;
1091        }
1092
1093        final String charSet;
1094        if (characterSet.isPresent())
1095        {
1096          charSet =
1097               characterSet.getValue().replace("]", "]]").replace("[", "[[");
1098        }
1099        else
1100        {
1101          charSet = "abcdefghijklmnopqrstuvwxyz";
1102        }
1103
1104        valuePatternString = "[random:" + length + ':' + charSet + ']';
1105      }
1106
1107      final ValuePattern parsedValuePattern;
1108      try
1109      {
1110        parsedValuePattern = new ValuePattern(valuePatternString);
1111      }
1112      catch (final ParseException e)
1113      {
1114        Debug.debugException(e);
1115        err(e.getMessage());
1116        return ResultCode.PARAM_ERROR;
1117      }
1118
1119      threads[i] = new ModRateThread(this, i, connection, dnPattern, attrs,
1120           parsedValuePattern, valueCount.getValue(), increment.isPresent(),
1121           incrementAmount.getValue(), controlArray, authzIDPattern,
1122           iterationsBeforeReconnect.getValue(), runningThreads, barrier,
1123           modCounter, modDurations, errorCounter, rcCounter, fixedRateBarrier);
1124      threads[i].start();
1125    }
1126
1127
1128    // Display the table header.
1129    for (final String headerLine : formatter.getHeaderLines(true))
1130    {
1131      out(headerLine);
1132    }
1133
1134
1135    // Start the RateAdjustor before the threads so that the initial value is
1136    // in place before any load is generated unless we're doing a warm-up in
1137    // which case, we'll start it after the warm-up is complete.
1138    if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0))
1139    {
1140      rateAdjustor.start();
1141    }
1142
1143
1144    // Indicate that the threads can start running.
1145    try
1146    {
1147      barrier.await();
1148    }
1149    catch (final Exception e)
1150    {
1151      Debug.debugException(e);
1152    }
1153
1154    long overallStartTime = System.nanoTime();
1155    long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis;
1156
1157
1158    boolean setOverallStartTime = false;
1159    long    lastDuration        = 0L;
1160    long    lastNumErrors       = 0L;
1161    long    lastNumMods         = 0L;
1162    long    lastEndTime         = System.nanoTime();
1163    for (long i=0; i < totalIntervals; i++)
1164    {
1165      if (rateAdjustor != null)
1166      {
1167        if (! rateAdjustor.isAlive())
1168        {
1169          out("All of the rates in " + variableRateData.getValue().getName() +
1170              " have been completed.");
1171          break;
1172        }
1173      }
1174
1175      final long startTimeMillis = System.currentTimeMillis();
1176      final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis;
1177      nextIntervalStartTime += intervalMillis;
1178      if (sleepTimeMillis > 0)
1179      {
1180        sleeper.sleep(sleepTimeMillis);
1181      }
1182
1183      if (stopRequested.get())
1184      {
1185        break;
1186      }
1187
1188      final long endTime          = System.nanoTime();
1189      final long intervalDuration = endTime - lastEndTime;
1190
1191      final long numMods;
1192      final long numErrors;
1193      final long totalDuration;
1194      if (warmUp && (remainingWarmUpIntervals > 0))
1195      {
1196        numMods       = modCounter.getAndSet(0L);
1197        numErrors     = errorCounter.getAndSet(0L);
1198        totalDuration = modDurations.getAndSet(0L);
1199      }
1200      else
1201      {
1202        numMods       = modCounter.get();
1203        numErrors     = errorCounter.get();
1204        totalDuration = modDurations.get();
1205      }
1206
1207      final long recentNumMods = numMods - lastNumMods;
1208      final long recentNumErrors = numErrors - lastNumErrors;
1209      final long recentDuration = totalDuration - lastDuration;
1210
1211      final double numSeconds = intervalDuration / 1_000_000_000.0d;
1212      final double recentModRate = recentNumMods / numSeconds;
1213      final double recentErrorRate  = recentNumErrors / numSeconds;
1214
1215      final double recentAvgDuration;
1216      if (recentNumMods > 0L)
1217      {
1218        recentAvgDuration = 1.0d * recentDuration / recentNumMods / 1_000_000;
1219      }
1220      else
1221      {
1222        recentAvgDuration = 0.0d;
1223      }
1224
1225      if (warmUp && (remainingWarmUpIntervals > 0))
1226      {
1227        out(formatter.formatRow(recentModRate, recentAvgDuration,
1228             recentErrorRate, "warming up", "warming up"));
1229
1230        remainingWarmUpIntervals--;
1231        if (remainingWarmUpIntervals == 0)
1232        {
1233          out("Warm-up completed.  Beginning overall statistics collection.");
1234          setOverallStartTime = true;
1235          if (rateAdjustor != null)
1236          {
1237            rateAdjustor.start();
1238          }
1239        }
1240      }
1241      else
1242      {
1243        if (setOverallStartTime)
1244        {
1245          overallStartTime    = lastEndTime;
1246          setOverallStartTime = false;
1247        }
1248
1249        final double numOverallSeconds =
1250             (endTime - overallStartTime) / 1_000_000_000.0d;
1251        final double overallAuthRate = numMods / numOverallSeconds;
1252
1253        final double overallAvgDuration;
1254        if (numMods > 0L)
1255        {
1256          overallAvgDuration = 1.0d * totalDuration / numMods / 1_000_000;
1257        }
1258        else
1259        {
1260          overallAvgDuration = 0.0d;
1261        }
1262
1263        out(formatter.formatRow(recentModRate, recentAvgDuration,
1264             recentErrorRate, overallAuthRate, overallAvgDuration));
1265
1266        lastNumMods     = numMods;
1267        lastNumErrors   = numErrors;
1268        lastDuration    = totalDuration;
1269      }
1270
1271      final List<ObjectPair<ResultCode,Long>> rcCounts =
1272           rcCounter.getCounts(true);
1273      if ((! suppressErrorsArgument.isPresent()) && (! rcCounts.isEmpty()))
1274      {
1275        err("\tError Results:");
1276        for (final ObjectPair<ResultCode,Long> p : rcCounts)
1277        {
1278          err("\t", p.getFirst().getName(), ":  ", p.getSecond());
1279        }
1280      }
1281
1282      lastEndTime = endTime;
1283    }
1284
1285    // Shut down the RateAdjustor if we have one.
1286    if (rateAdjustor != null)
1287    {
1288      rateAdjustor.shutDown();
1289    }
1290
1291    // Stop all of the threads.
1292    ResultCode resultCode = ResultCode.SUCCESS;
1293    for (final ModRateThread t : threads)
1294    {
1295      final ResultCode r = t.stopRunning();
1296      if (resultCode == ResultCode.SUCCESS)
1297      {
1298        resultCode = r;
1299      }
1300    }
1301
1302    return resultCode;
1303  }
1304
1305
1306
1307  /**
1308   * Requests that this tool stop running.  This method will attempt to wait
1309   * for all threads to complete before returning control to the caller.
1310   */
1311  public void stopRunning()
1312  {
1313    stopRequested.set(true);
1314    sleeper.wakeup();
1315
1316    while (true)
1317    {
1318      final int stillRunning = runningThreads.get();
1319      if (stillRunning <= 0)
1320      {
1321        break;
1322      }
1323      else
1324      {
1325        try
1326        {
1327          Thread.sleep(1L);
1328        } catch (final Exception e) {}
1329      }
1330    }
1331  }
1332
1333
1334
1335  /**
1336   * {@inheritDoc}
1337   */
1338  @Override()
1339  public LinkedHashMap<String[],String> getExampleUsages()
1340  {
1341    final LinkedHashMap<String[],String> examples =
1342         new LinkedHashMap<>(StaticUtils.computeMapCapacity(2));
1343
1344    String[] args =
1345    {
1346      "--hostname", "server.example.com",
1347      "--port", "389",
1348      "--bindDN", "uid=admin,dc=example,dc=com",
1349      "--bindPassword", "password",
1350      "--entryDN", "uid=user.[1-1000000],ou=People,dc=example,dc=com",
1351      "--attribute", "description",
1352      "--valueLength", "12",
1353      "--numThreads", "10"
1354    };
1355    String description =
1356         "Test modify performance by randomly selecting entries across a set " +
1357         "of one million users located below 'ou=People,dc=example,dc=com' " +
1358         "with ten concurrent threads and replacing the values for the " +
1359         "description attribute with a string of 12 randomly-selected " +
1360         "lowercase alphabetic characters.";
1361    examples.put(args, description);
1362
1363    args = new String[]
1364    {
1365      "--generateSampleRateFile", "variable-rate-data.txt"
1366    };
1367    description =
1368         "Generate a sample variable rate definition file that may be used " +
1369         "in conjunction with the --variableRateData argument.  The sample " +
1370         "file will include comments that describe the format for data to be " +
1371         "included in this file.";
1372    examples.put(args, description);
1373
1374    return examples;
1375  }
1376}