001/*
002 * Copyright 2008-2019 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright (C) 2008-2019 Ping Identity Corporation
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021package com.unboundid.ldap.sdk.examples;
022
023
024
025import java.io.File;
026import java.io.FileInputStream;
027import java.io.InputStream;
028import java.io.IOException;
029import java.io.OutputStream;
030import java.util.ArrayList;
031import java.util.Iterator;
032import java.util.TreeMap;
033import java.util.LinkedHashMap;
034import java.util.List;
035import java.util.concurrent.atomic.AtomicLong;
036import java.util.zip.GZIPInputStream;
037
038import com.unboundid.ldap.sdk.Entry;
039import com.unboundid.ldap.sdk.LDAPConnection;
040import com.unboundid.ldap.sdk.LDAPException;
041import com.unboundid.ldap.sdk.ResultCode;
042import com.unboundid.ldap.sdk.Version;
043import com.unboundid.ldap.sdk.schema.Schema;
044import com.unboundid.ldap.sdk.schema.EntryValidator;
045import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils;
046import com.unboundid.ldif.DuplicateValueBehavior;
047import com.unboundid.ldif.LDIFException;
048import com.unboundid.ldif.LDIFReader;
049import com.unboundid.ldif.LDIFReaderEntryTranslator;
050import com.unboundid.ldif.LDIFWriter;
051import com.unboundid.util.Debug;
052import com.unboundid.util.LDAPCommandLineTool;
053import com.unboundid.util.StaticUtils;
054import com.unboundid.util.ThreadSafety;
055import com.unboundid.util.ThreadSafetyLevel;
056import com.unboundid.util.args.ArgumentException;
057import com.unboundid.util.args.ArgumentParser;
058import com.unboundid.util.args.BooleanArgument;
059import com.unboundid.util.args.FileArgument;
060import com.unboundid.util.args.IntegerArgument;
061import com.unboundid.util.args.StringArgument;
062
063
064
065/**
066 * This class provides a simple tool that can be used to validate that the
067 * contents of an LDIF file are valid.  This includes ensuring that the contents
068 * can be parsed as valid LDIF, and it can also ensure that the LDIF content
069 * conforms to the server schema.  It will obtain the schema by connecting to
070 * the server and retrieving the default schema (i.e., the schema which governs
071 * the root DSE).  By default, a thorough set of validation will be performed,
072 * but it is possible to disable certain types of validation.
073 * <BR><BR>
074 * Some of the APIs demonstrated by this example include:
075 * <UL>
076 *   <LI>Argument Parsing (from the {@code com.unboundid.util.args}
077 *       package)</LI>
078 *   <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util}
079 *       package)</LI>
080 *   <LI>LDIF Processing (from the {@code com.unboundid.ldif} package)</LI>
081 *   <LI>Schema Parsing (from the {@code com.unboundid.ldap.sdk.schema}
082 *       package)</LI>
083 * </UL>
084 * <BR><BR>
085 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
086 * class (to obtain the information to use to connect to the server to read the
087 * schema), as well as the following additional arguments:
088 * <UL>
089 *   <LI>"--schemaDirectory {path}" -- specifies the path to a directory
090 *       containing files with schema definitions.  If this argument is
091 *       provided, then no attempt will be made to communicate with a directory
092 *       server.</LI>
093 *   <LI>"-f {path}" or "--ldifFile {path}" -- specifies the path to the LDIF
094 *       file to be validated.</LI>
095 *   <LI>"-c" or "--isCompressed" -- indicates that the LDIF file is
096 *       compressed.</LI>
097 *   <LI>"-R {path}" or "--rejectFile {path}" -- specifies the path to the file
098 *       to be written with information about all entries that failed
099 *       validation.</LI>
100 *   <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of
101 *       concurrent threads to use when processing the LDIF.  If this is not
102 *       provided, then a default of one thread will be used.</LI>
103 *   <LI>"--ignoreUndefinedObjectClasses" -- indicates that the validation
104 *       process should ignore validation failures due to entries that contain
105 *       object classes not defined in the server schema.</LI>
106 *   <LI>"--ignoreUndefinedAttributes" -- indicates that the validation process
107 *       should ignore validation failures due to entries that contain
108 *       attributes not defined in the server schema.</LI>
109 *   <LI>"--ignoreMalformedDNs" -- indicates that the validation process should
110 *       ignore validation failures due to entries with malformed DNs.</LI>
111 *   <LI>"--ignoreMissingRDNValues" -- indicates that the validation process
112 *       should ignore validation failures due to entries that contain an RDN
113 *       attribute value that is not present in the set of entry
114 *       attributes.</LI>
115 *   <LI>"--ignoreStructuralObjectClasses" -- indicates that the validation
116 *       process should ignore validation failures due to entries that either do
117 *       not have a structural object class or that have multiple structural
118 *       object classes.</LI>
119 *   <LI>"--ignoreProhibitedObjectClasses" -- indicates that the validation
120 *       process should ignore validation failures due to entries containing
121 *       auxiliary classes that are not allowed by a DIT content rule, or
122 *       abstract classes that are not subclassed by an auxiliary or structural
123 *       class contained in the entry.</LI>
124 *   <LI>"--ignoreProhibitedAttributes" -- indicates that the validation process
125 *       should ignore validation failures due to entries including attributes
126 *       that are not allowed or are explicitly prohibited by a DIT content
127 *       rule.</LI>
128 *   <LI>"--ignoreMissingAttributes" -- indicates that the validation process
129 *       should ignore validation failures due to entries missing required
130 *       attributes.</LI>
131 *   <LI>"--ignoreSingleValuedAttributes" -- indicates that the validation
132 *       process should ignore validation failures due to single-valued
133 *       attributes containing multiple values.</LI>
134 *   <LI>"--ignoreAttributeSyntax" -- indicates that the validation process
135 *       should ignore validation failures due to attribute values which violate
136 *       the associated attribute syntax.</LI>
137 *   <LI>"--ignoreSyntaxViolationsForAttribute" -- indicates that the validation
138 *       process should ignore validation failures due to attribute values which
139 *       violate the associated attribute syntax, but only for the specified
140 *       attribute types.</LI>
141 *   <LI>"--ignoreNameForms" -- indicates that the validation process should
142 *       ignore validation failures due to name form violations (in which the
143 *       entry's RDN does not comply with the associated name form).</LI>
144 * </UL>
145 */
146@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
147public final class ValidateLDIF
148       extends LDAPCommandLineTool
149       implements LDIFReaderEntryTranslator
150{
151  /**
152   * The end-of-line character for this platform.
153   */
154  private static final String EOL =
155       StaticUtils.getSystemProperty("line.separator", "\n");
156
157
158
159  // The arguments used by this program.
160  private BooleanArgument ignoreDuplicateValues;
161  private BooleanArgument ignoreUndefinedObjectClasses;
162  private BooleanArgument ignoreUndefinedAttributes;
163  private BooleanArgument ignoreMalformedDNs;
164  private BooleanArgument ignoreMissingRDNValues;
165  private BooleanArgument ignoreMissingSuperiorObjectClasses;
166  private BooleanArgument ignoreStructuralObjectClasses;
167  private BooleanArgument ignoreProhibitedObjectClasses;
168  private BooleanArgument ignoreProhibitedAttributes;
169  private BooleanArgument ignoreMissingAttributes;
170  private BooleanArgument ignoreSingleValuedAttributes;
171  private BooleanArgument ignoreAttributeSyntax;
172  private BooleanArgument ignoreNameForms;
173  private BooleanArgument isCompressed;
174  private FileArgument    schemaDirectory;
175  private FileArgument    ldifFile;
176  private FileArgument    rejectFile;
177  private FileArgument    encryptionPassphraseFile;
178  private IntegerArgument numThreads;
179  private StringArgument  ignoreSyntaxViolationsForAttribute;
180
181  // The counter used to keep track of the number of entries processed.
182  private final AtomicLong entriesProcessed = new AtomicLong(0L);
183
184  // The counter used to keep track of the number of entries that could not be
185  // parsed as valid entries.
186  private final AtomicLong malformedEntries = new AtomicLong(0L);
187
188  // The entry validator that will be used to validate the entries.
189  private EntryValidator entryValidator;
190
191  // The LDIF writer that will be used to write rejected entries.
192  private LDIFWriter rejectWriter;
193
194
195
196  /**
197   * Parse the provided command line arguments and make the appropriate set of
198   * changes.
199   *
200   * @param  args  The command line arguments provided to this program.
201   */
202  public static void main(final String[] args)
203  {
204    final ResultCode resultCode = main(args, System.out, System.err);
205    if (resultCode != ResultCode.SUCCESS)
206    {
207      System.exit(resultCode.intValue());
208    }
209  }
210
211
212
213  /**
214   * Parse the provided command line arguments and make the appropriate set of
215   * changes.
216   *
217   * @param  args       The command line arguments provided to this program.
218   * @param  outStream  The output stream to which standard out should be
219   *                    written.  It may be {@code null} if output should be
220   *                    suppressed.
221   * @param  errStream  The output stream to which standard error should be
222   *                    written.  It may be {@code null} if error messages
223   *                    should be suppressed.
224   *
225   * @return  A result code indicating whether the processing was successful.
226   */
227  public static ResultCode main(final String[] args,
228                                final OutputStream outStream,
229                                final OutputStream errStream)
230  {
231    final ValidateLDIF validateLDIF = new ValidateLDIF(outStream, errStream);
232    return validateLDIF.runTool(args);
233  }
234
235
236
237  /**
238   * Creates a new instance of this tool.
239   *
240   * @param  outStream  The output stream to which standard out should be
241   *                    written.  It may be {@code null} if output should be
242   *                    suppressed.
243   * @param  errStream  The output stream to which standard error should be
244   *                    written.  It may be {@code null} if error messages
245   *                    should be suppressed.
246   */
247  public ValidateLDIF(final OutputStream outStream,
248                      final OutputStream errStream)
249  {
250    super(outStream, errStream);
251  }
252
253
254
255  /**
256   * Retrieves the name for this tool.
257   *
258   * @return  The name for this tool.
259   */
260  @Override()
261  public String getToolName()
262  {
263    return "validate-ldif";
264  }
265
266
267
268  /**
269   * Retrieves the description for this tool.
270   *
271   * @return  The description for this tool.
272   */
273  @Override()
274  public String getToolDescription()
275  {
276    return "Validate the contents of an LDIF file " +
277           "against the server schema.";
278  }
279
280
281
282  /**
283   * Retrieves the version string for this tool.
284   *
285   * @return  The version string for this tool.
286   */
287  @Override()
288  public String getToolVersion()
289  {
290    return Version.NUMERIC_VERSION_STRING;
291  }
292
293
294
295  /**
296   * Indicates whether this tool should provide support for an interactive mode,
297   * in which the tool offers a mode in which the arguments can be provided in
298   * a text-driven menu rather than requiring them to be given on the command
299   * line.  If interactive mode is supported, it may be invoked using the
300   * "--interactive" argument.  Alternately, if interactive mode is supported
301   * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
302   * interactive mode may be invoked by simply launching the tool without any
303   * arguments.
304   *
305   * @return  {@code true} if this tool supports interactive mode, or
306   *          {@code false} if not.
307   */
308  @Override()
309  public boolean supportsInteractiveMode()
310  {
311    return true;
312  }
313
314
315
316  /**
317   * Indicates whether this tool defaults to launching in interactive mode if
318   * the tool is invoked without any command-line arguments.  This will only be
319   * used if {@link #supportsInteractiveMode()} returns {@code true}.
320   *
321   * @return  {@code true} if this tool defaults to using interactive mode if
322   *          launched without any command-line arguments, or {@code false} if
323   *          not.
324   */
325  @Override()
326  public boolean defaultsToInteractiveMode()
327  {
328    return true;
329  }
330
331
332
333  /**
334   * Indicates whether this tool should provide arguments for redirecting output
335   * to a file.  If this method returns {@code true}, then the tool will offer
336   * an "--outputFile" argument that will specify the path to a file to which
337   * all standard output and standard error content will be written, and it will
338   * also offer a "--teeToStandardOut" argument that can only be used if the
339   * "--outputFile" argument is present and will cause all output to be written
340   * to both the specified output file and to standard output.
341   *
342   * @return  {@code true} if this tool should provide arguments for redirecting
343   *          output to a file, or {@code false} if not.
344   */
345  @Override()
346  protected boolean supportsOutputFile()
347  {
348    return true;
349  }
350
351
352
353  /**
354   * Indicates whether this tool should default to interactively prompting for
355   * the bind password if a password is required but no argument was provided
356   * to indicate how to get the password.
357   *
358   * @return  {@code true} if this tool should default to interactively
359   *          prompting for the bind password, or {@code false} if not.
360   */
361  @Override()
362  protected boolean defaultToPromptForBindPassword()
363  {
364    return true;
365  }
366
367
368
369  /**
370   * Indicates whether this tool supports the use of a properties file for
371   * specifying default values for arguments that aren't specified on the
372   * command line.
373   *
374   * @return  {@code true} if this tool supports the use of a properties file
375   *          for specifying default values for arguments that aren't specified
376   *          on the command line, or {@code false} if not.
377   */
378  @Override()
379  public boolean supportsPropertiesFile()
380  {
381    return true;
382  }
383
384
385
386  /**
387   * Indicates whether the LDAP-specific arguments should include alternate
388   * versions of all long identifiers that consist of multiple words so that
389   * they are available in both camelCase and dash-separated versions.
390   *
391   * @return  {@code true} if this tool should provide multiple versions of
392   *          long identifiers for LDAP-specific arguments, or {@code false} if
393   *          not.
394   */
395  @Override()
396  protected boolean includeAlternateLongIdentifiers()
397  {
398    return true;
399  }
400
401
402
403  /**
404   * Indicates whether this tool should provide a command-line argument that
405   * allows for low-level SSL debugging.  If this returns {@code true}, then an
406   * "--enableSSLDebugging}" argument will be added that sets the
407   * "javax.net.debug" system property to "all" before attempting any
408   * communication.
409   *
410   * @return  {@code true} if this tool should offer an "--enableSSLDebugging"
411   *          argument, or {@code false} if not.
412   */
413  @Override()
414  protected boolean supportsSSLDebugging()
415  {
416    return true;
417  }
418
419
420
421  /**
422   * Adds the arguments used by this program that aren't already provided by the
423   * generic {@code LDAPCommandLineTool} framework.
424   *
425   * @param  parser  The argument parser to which the arguments should be added.
426   *
427   * @throws  ArgumentException  If a problem occurs while adding the arguments.
428   */
429  @Override()
430  public void addNonLDAPArguments(final ArgumentParser parser)
431         throws ArgumentException
432  {
433    String description = "The path to the LDIF file to process.  The tool " +
434         "will automatically attempt to detect whether the file is " +
435         "encrypted or compressed.";
436    ldifFile = new FileArgument('f', "ldifFile", true, 1, "{path}", description,
437                                true, true, true, false);
438    ldifFile.addLongIdentifier("ldif-file", true);
439    parser.addArgument(ldifFile);
440
441
442    // Add an argument that makes it possible to read a compressed LDIF file.
443    // Note that this argument is no longer needed for dealing with compressed
444    // files, since the tool will automatically detect whether a file is
445    // compressed.  However, the argument is still provided for the purpose of
446    // backward compatibility.
447    description = "Indicates that the specified LDIF file is compressed " +
448                  "using gzip compression.";
449    isCompressed = new BooleanArgument('c', "isCompressed", description);
450    isCompressed.addLongIdentifier("is-compressed", true);
451    isCompressed.setHidden(true);
452    parser.addArgument(isCompressed);
453
454
455    // Add an argument that indicates that the tool should read the encryption
456    // passphrase from a file.
457    description = "Indicates that the specified LDIF file is encrypted and " +
458         "that the encryption passphrase is contained in the specified " +
459         "file.  If the LDIF data is encrypted and this argument is not " +
460         "provided, then the tool will interactively prompt for the " +
461         "encryption passphrase.";
462    encryptionPassphraseFile = new FileArgument(null,
463         "encryptionPassphraseFile", false, 1, null, description, true, true,
464         true, false);
465    encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file",
466         true);
467    encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true);
468    encryptionPassphraseFile.addLongIdentifier("encryption-password-file",
469         true);
470    parser.addArgument(encryptionPassphraseFile);
471
472
473    description = "The path to the file to which rejected entries should be " +
474                  "written.";
475    rejectFile = new FileArgument('R', "rejectFile", false, 1, "{path}",
476                                  description, false, true, true, false);
477    rejectFile.addLongIdentifier("reject-file", true);
478    parser.addArgument(rejectFile);
479
480    description = "The path to a directory containing one or more LDIF files " +
481                  "with the schema information to use.  If this is provided, " +
482                  "then no LDAP communication will be performed.";
483    schemaDirectory = new FileArgument(null, "schemaDirectory", false, 1,
484         "{path}", description, true, true, false, true);
485    schemaDirectory.addLongIdentifier("schema-directory", true);
486    parser.addArgument(schemaDirectory);
487
488    description = "The number of threads to use when processing the LDIF file.";
489    numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}",
490         description, 1, Integer.MAX_VALUE, 1);
491    numThreads.addLongIdentifier("num-threads", true);
492    parser.addArgument(numThreads);
493
494    description = "Ignore validation failures due to entries containing " +
495                  "duplicate values for the same attribute.";
496    ignoreDuplicateValues =
497         new BooleanArgument(null, "ignoreDuplicateValues", description);
498    ignoreDuplicateValues.setArgumentGroupName(
499         "Validation Strictness Arguments");
500    ignoreDuplicateValues.addLongIdentifier("ignore-duplicate-values", true);
501    parser.addArgument(ignoreDuplicateValues);
502
503    description = "Ignore validation failures due to object classes not " +
504                  "defined in the schema.";
505    ignoreUndefinedObjectClasses =
506         new BooleanArgument(null, "ignoreUndefinedObjectClasses", description);
507    ignoreUndefinedObjectClasses.setArgumentGroupName(
508         "Validation Strictness Arguments");
509    ignoreUndefinedObjectClasses.addLongIdentifier(
510         "ignore-undefined-object-classes", true);
511    parser.addArgument(ignoreUndefinedObjectClasses);
512
513    description = "Ignore validation failures due to attributes not defined " +
514                  "in the schema.";
515    ignoreUndefinedAttributes =
516         new BooleanArgument(null, "ignoreUndefinedAttributes", description);
517    ignoreUndefinedAttributes.setArgumentGroupName(
518         "Validation Strictness Arguments");
519    ignoreUndefinedAttributes.addLongIdentifier("ignore-undefined-attributes",
520         true);
521    parser.addArgument(ignoreUndefinedAttributes);
522
523    description = "Ignore validation failures due to entries with malformed " +
524                  "DNs.";
525    ignoreMalformedDNs =
526         new BooleanArgument(null, "ignoreMalformedDNs", description);
527    ignoreMalformedDNs.setArgumentGroupName("Validation Strictness Arguments");
528    ignoreMalformedDNs.addLongIdentifier("ignore-malformed-dns", true);
529    parser.addArgument(ignoreMalformedDNs);
530
531    description = "Ignore validation failures due to entries with RDN " +
532                  "attribute values that are missing from the set of entry " +
533                  "attributes.";
534    ignoreMissingRDNValues =
535         new BooleanArgument(null, "ignoreMissingRDNValues", description);
536    ignoreMissingRDNValues.setArgumentGroupName(
537         "Validation Strictness Arguments");
538    ignoreMissingRDNValues.addLongIdentifier("ignore-missing-rdn-values", true);
539    parser.addArgument(ignoreMissingRDNValues);
540
541    description = "Ignore validation failures due to entries without exactly " +
542                  "structural object class.";
543    ignoreStructuralObjectClasses =
544         new BooleanArgument(null, "ignoreStructuralObjectClasses",
545                             description);
546    ignoreStructuralObjectClasses.setArgumentGroupName(
547         "Validation Strictness Arguments");
548    ignoreStructuralObjectClasses.addLongIdentifier(
549         "ignore-structural-object-classes", true);
550    parser.addArgument(ignoreStructuralObjectClasses);
551
552    description = "Ignore validation failures due to entries with object " +
553                  "classes that are not allowed.";
554    ignoreProhibitedObjectClasses =
555         new BooleanArgument(null, "ignoreProhibitedObjectClasses",
556                             description);
557    ignoreProhibitedObjectClasses.setArgumentGroupName(
558         "Validation Strictness Arguments");
559    ignoreProhibitedObjectClasses.addLongIdentifier(
560         "ignore-prohibited-object-classes", true);
561    parser.addArgument(ignoreProhibitedObjectClasses);
562
563    description = "Ignore validation failures due to entries that are " +
564                  "one or more superior object classes.";
565    ignoreMissingSuperiorObjectClasses =
566         new BooleanArgument(null, "ignoreMissingSuperiorObjectClasses",
567              description);
568    ignoreMissingSuperiorObjectClasses.setArgumentGroupName(
569         "Validation Strictness Arguments");
570    ignoreMissingSuperiorObjectClasses.addLongIdentifier(
571         "ignore-missing-superior-object-classes", true);
572    parser.addArgument(ignoreMissingSuperiorObjectClasses);
573
574    description = "Ignore validation failures due to entries with attributes " +
575                  "that are not allowed.";
576    ignoreProhibitedAttributes =
577         new BooleanArgument(null, "ignoreProhibitedAttributes", description);
578    ignoreProhibitedAttributes.setArgumentGroupName(
579         "Validation Strictness Arguments");
580    ignoreProhibitedAttributes.addLongIdentifier(
581         "ignore-prohibited-attributes", true);
582    parser.addArgument(ignoreProhibitedAttributes);
583
584    description = "Ignore validation failures due to entries missing " +
585                  "required attributes.";
586    ignoreMissingAttributes =
587         new BooleanArgument(null, "ignoreMissingAttributes", description);
588    ignoreMissingAttributes.setArgumentGroupName(
589         "Validation Strictness Arguments");
590    ignoreMissingAttributes.addLongIdentifier("ignore-missing-attributes",
591         true);
592    parser.addArgument(ignoreMissingAttributes);
593
594    description = "Ignore validation failures due to entries with multiple " +
595                  "values for single-valued attributes.";
596    ignoreSingleValuedAttributes =
597         new BooleanArgument(null, "ignoreSingleValuedAttributes", description);
598    ignoreSingleValuedAttributes.setArgumentGroupName(
599         "Validation Strictness Arguments");
600    ignoreSingleValuedAttributes.addLongIdentifier(
601         "ignore-single-valued-attributes", true);
602    parser.addArgument(ignoreSingleValuedAttributes);
603
604    description = "Ignore validation failures due to entries with attribute " +
605                  "values that violate their associated syntax.  If this is " +
606                  "provided, then no attribute syntax violations will be " +
607                  "flagged.  If this is not provided, then all attribute " +
608                  "syntax violations will be flagged except for violations " +
609                  "in those attributes excluded by the " +
610                  "--ignoreSyntaxViolationsForAttribute argument.";
611    ignoreAttributeSyntax =
612         new BooleanArgument(null, "ignoreAttributeSyntax", description);
613    ignoreAttributeSyntax.setArgumentGroupName(
614         "Validation Strictness Arguments");
615    ignoreAttributeSyntax.addLongIdentifier("ignore-attribute-syntax", true);
616    parser.addArgument(ignoreAttributeSyntax);
617
618    description = "The name or OID of an attribute for which to ignore " +
619                  "validation failures due to violations of the associated " +
620                  "attribute syntax.  This argument can only be used if the " +
621                  "--ignoreAttributeSyntax argument is not provided.";
622    ignoreSyntaxViolationsForAttribute = new StringArgument(null,
623         "ignoreSyntaxViolationsForAttribute", false, 0, "{attr}", description);
624    ignoreSyntaxViolationsForAttribute.setArgumentGroupName(
625         "Validation Strictness Arguments");
626    ignoreSyntaxViolationsForAttribute.addLongIdentifier(
627         "ignore-syntax-violations-for-attribute", true);
628    parser.addArgument(ignoreSyntaxViolationsForAttribute);
629
630    description = "Ignore validation failures due to entries with RDNs " +
631                  "that violate the associated name form definition.";
632    ignoreNameForms = new BooleanArgument(null, "ignoreNameForms", description);
633    ignoreNameForms.setArgumentGroupName("Validation Strictness Arguments");
634    ignoreNameForms.addLongIdentifier("ignore-name-forms", true);
635    parser.addArgument(ignoreNameForms);
636
637
638    // The ignoreAttributeSyntax and ignoreAttributeSyntaxForAttribute arguments
639    // cannot be used together.
640    parser.addExclusiveArgumentSet(ignoreAttributeSyntax,
641         ignoreSyntaxViolationsForAttribute);
642  }
643
644
645
646  /**
647   * Performs the actual processing for this tool.  In this case, it gets a
648   * connection to the directory server and uses it to retrieve the server
649   * schema.  It then reads the LDIF file and validates each entry accordingly.
650   *
651   * @return  The result code for the processing that was performed.
652   */
653  @Override()
654  public ResultCode doToolProcessing()
655  {
656    // Get the connection to the directory server and use it to read the schema.
657    final Schema schema;
658    if (schemaDirectory.isPresent())
659    {
660      final File schemaDir = schemaDirectory.getValue();
661
662      try
663      {
664        final TreeMap<String,File> fileMap = new TreeMap<>();
665        for (final File f : schemaDir.listFiles())
666        {
667          final String name = f.getName();
668          if (f.isFile() && name.endsWith(".ldif"))
669          {
670            fileMap.put(name, f);
671          }
672        }
673
674        if (fileMap.isEmpty())
675        {
676          err("No LDIF files found in directory " +
677              schemaDir.getAbsolutePath());
678          return ResultCode.PARAM_ERROR;
679        }
680
681        final ArrayList<File> fileList = new ArrayList<>(fileMap.values());
682        schema = Schema.getSchema(fileList);
683      }
684      catch (final Exception e)
685      {
686        Debug.debugException(e);
687        err("Unable to read schema from files in directory " +
688            schemaDir.getAbsolutePath() + ":  " +
689             StaticUtils.getExceptionMessage(e));
690        return ResultCode.LOCAL_ERROR;
691      }
692    }
693    else
694    {
695      try
696      {
697        final LDAPConnection connection = getConnection();
698        schema = connection.getSchema();
699        connection.close();
700      }
701      catch (final LDAPException le)
702      {
703        Debug.debugException(le);
704        err("Unable to connect to the directory server and read the schema:  ",
705            le.getMessage());
706        return le.getResultCode();
707      }
708    }
709
710
711    // Get the encryption passphrase, if it was provided.
712    String encryptionPassphrase = null;
713    if (encryptionPassphraseFile.isPresent())
714    {
715      try
716      {
717        encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile(
718             encryptionPassphraseFile.getValue());
719      }
720      catch (final LDAPException e)
721      {
722        Debug.debugException(e);
723        err(e.getMessage());
724        return e.getResultCode();
725      }
726    }
727
728
729    // Create the entry validator and initialize its configuration.
730    entryValidator = new EntryValidator(schema);
731    entryValidator.setCheckAttributeSyntax(!ignoreAttributeSyntax.isPresent());
732    entryValidator.setCheckMalformedDNs(!ignoreMalformedDNs.isPresent());
733    entryValidator.setCheckEntryMissingRDNValues(
734         !ignoreMissingRDNValues.isPresent());
735    entryValidator.setCheckMissingAttributes(
736         !ignoreMissingAttributes.isPresent());
737    entryValidator.setCheckNameForms(!ignoreNameForms.isPresent());
738    entryValidator.setCheckProhibitedAttributes(
739         !ignoreProhibitedAttributes.isPresent());
740    entryValidator.setCheckProhibitedObjectClasses(
741         !ignoreProhibitedObjectClasses.isPresent());
742    entryValidator.setCheckMissingSuperiorObjectClasses(
743         !ignoreMissingSuperiorObjectClasses.isPresent());
744    entryValidator.setCheckSingleValuedAttributes(
745         !ignoreSingleValuedAttributes.isPresent());
746    entryValidator.setCheckStructuralObjectClasses(
747         !ignoreStructuralObjectClasses.isPresent());
748    entryValidator.setCheckUndefinedAttributes(
749         !ignoreUndefinedAttributes.isPresent());
750    entryValidator.setCheckUndefinedObjectClasses(
751         !ignoreUndefinedObjectClasses.isPresent());
752
753    if (ignoreSyntaxViolationsForAttribute.isPresent())
754    {
755      entryValidator.setIgnoreSyntaxViolationAttributeTypes(
756           ignoreSyntaxViolationsForAttribute.getValues());
757    }
758
759
760    // Create an LDIF reader that can be used to read through the LDIF file.
761    final LDIFReader ldifReader;
762    rejectWriter = null;
763    try
764    {
765      InputStream inputStream = new FileInputStream(ldifFile.getValue());
766
767      inputStream = ToolUtils.getPossiblyPassphraseEncryptedInputStream(
768           inputStream, encryptionPassphrase, false,
769           "LDIF file '" + ldifFile.getValue().getPath() +
770                "' is encrypted.  Please enter the encryption passphrase:",
771             "ERROR:  The provided passphrase was incorrect.",
772             getOut(), getErr()).getFirst();
773
774      if (isCompressed.isPresent())
775      {
776        inputStream = new GZIPInputStream(inputStream);
777      }
778      else
779      {
780        inputStream =
781             ToolUtils.getPossiblyGZIPCompressedInputStream(inputStream);
782      }
783
784      ldifReader = new LDIFReader(inputStream, numThreads.getValue(), this);
785    }
786    catch (final Exception e)
787    {
788      Debug.debugException(e);
789      err("Unable to open the LDIF reader:  ",
790           StaticUtils.getExceptionMessage(e));
791      return ResultCode.LOCAL_ERROR;
792    }
793
794    ldifReader.setSchema(schema);
795    if (ignoreDuplicateValues.isPresent())
796    {
797      ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.STRIP);
798    }
799    else
800    {
801      ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.REJECT);
802    }
803
804    try
805    {
806      // Create an LDIF writer that can be used to write information about
807      // rejected entries.
808      try
809      {
810        if (rejectFile.isPresent())
811        {
812          rejectWriter = new LDIFWriter(rejectFile.getValue());
813        }
814      }
815      catch (final Exception e)
816      {
817        Debug.debugException(e);
818        err("Unable to create the reject writer:  ",
819             StaticUtils.getExceptionMessage(e));
820        return ResultCode.LOCAL_ERROR;
821      }
822
823      ResultCode resultCode = ResultCode.SUCCESS;
824      while (true)
825      {
826        try
827        {
828          final Entry e = ldifReader.readEntry();
829          if (e == null)
830          {
831            // Because we're performing parallel processing and returning null
832            // from the translate method, LDIFReader.readEntry() should never
833            // return a non-null value.  However, it can throw an LDIFException
834            // if it encounters an invalid entry, or an IOException if there's
835            // a problem reading from the file, so we should still iterate
836            // through all of the entries to catch and report on those problems.
837            break;
838          }
839        }
840        catch (final LDIFException le)
841        {
842          Debug.debugException(le);
843          malformedEntries.incrementAndGet();
844
845          if (resultCode == ResultCode.SUCCESS)
846          {
847            resultCode = ResultCode.DECODING_ERROR;
848          }
849
850          if (rejectWriter != null)
851          {
852            try
853            {
854              rejectWriter.writeComment(
855                   "Unable to parse an entry read from LDIF:", false, false);
856              if (le.mayContinueReading())
857              {
858                rejectWriter.writeComment(
859                     StaticUtils.getExceptionMessage(le), false, true);
860              }
861              else
862              {
863                rejectWriter.writeComment(
864                     StaticUtils.getExceptionMessage(le), false,
865                     false);
866                rejectWriter.writeComment("Unable to continue LDIF processing.",
867                     false, true);
868                err("Aborting LDIF processing:  ",
869                     StaticUtils.getExceptionMessage(le));
870                return ResultCode.LOCAL_ERROR;
871              }
872            }
873            catch (final IOException ioe)
874            {
875              Debug.debugException(ioe);
876              err("Unable to write to the reject file:",
877                  StaticUtils.getExceptionMessage(ioe));
878              err("LDIF parse failure that triggered the rejection:  ",
879                  StaticUtils.getExceptionMessage(le));
880              return ResultCode.LOCAL_ERROR;
881            }
882          }
883        }
884        catch (final IOException ioe)
885        {
886          Debug.debugException(ioe);
887
888          if (rejectWriter != null)
889          {
890            try
891            {
892              rejectWriter.writeComment("I/O error reading from LDIF:", false,
893                   false);
894              rejectWriter.writeComment(StaticUtils.getExceptionMessage(ioe),
895                   false, true);
896              return ResultCode.LOCAL_ERROR;
897            }
898            catch (final Exception ex)
899            {
900              Debug.debugException(ex);
901              err("I/O error reading from LDIF:",
902                   StaticUtils.getExceptionMessage(ioe));
903              return ResultCode.LOCAL_ERROR;
904            }
905          }
906        }
907      }
908
909      if (malformedEntries.get() > 0)
910      {
911        out(malformedEntries.get() + " entries were malformed and could not " +
912            "be read from the LDIF file.");
913      }
914
915      if (entryValidator.getInvalidEntries() > 0)
916      {
917        if (resultCode == ResultCode.SUCCESS)
918        {
919          resultCode = ResultCode.OBJECT_CLASS_VIOLATION;
920        }
921
922        for (final String s : entryValidator.getInvalidEntrySummary(true))
923        {
924          out(s);
925        }
926      }
927      else
928      {
929        if (malformedEntries.get() == 0)
930        {
931          out("No errors were encountered.");
932        }
933      }
934
935      return resultCode;
936    }
937    finally
938    {
939      try
940      {
941        ldifReader.close();
942      }
943      catch (final Exception e)
944      {
945        Debug.debugException(e);
946      }
947
948      try
949      {
950        if (rejectWriter != null)
951        {
952          rejectWriter.close();
953        }
954      }
955      catch (final Exception e)
956      {
957        Debug.debugException(e);
958      }
959    }
960  }
961
962
963
964  /**
965   * Examines the provided entry to determine whether it conforms to the
966   * server schema.
967   *
968   * @param  entry           The entry to be examined.
969   * @param  firstLineNumber The line number of the LDIF source on which the
970   *                         provided entry begins.
971   *
972   * @return  The updated entry.  This method will always return {@code null}
973   *          because all of the real processing needed for the entry is
974   *          performed in this method and the entry isn't needed any more
975   *          after this method is done.
976   */
977  @Override()
978  public Entry translate(final Entry entry, final long firstLineNumber)
979  {
980    final ArrayList<String> invalidReasons = new ArrayList<>(5);
981    if (! entryValidator.entryIsValid(entry, invalidReasons))
982    {
983      if (rejectWriter != null)
984      {
985        synchronized (this)
986        {
987          try
988          {
989            rejectWriter.writeEntry(entry, listToString(invalidReasons));
990          }
991          catch (final IOException ioe)
992          {
993            Debug.debugException(ioe);
994          }
995        }
996      }
997    }
998
999    final long numEntries = entriesProcessed.incrementAndGet();
1000    if ((numEntries % 1000L) == 0L)
1001    {
1002      out("Processed ", numEntries, " entries.");
1003    }
1004
1005    return null;
1006  }
1007
1008
1009
1010  /**
1011   * Converts the provided list of strings into a single string.  It will
1012   * contain line breaks after all but the last element.
1013   *
1014   * @param  l  The list of strings to convert to a single string.
1015   *
1016   * @return  The string from the provided list, or {@code null} if the provided
1017   *          list is empty or {@code null}.
1018   */
1019  private static String listToString(final List<String> l)
1020  {
1021    if ((l == null) || (l.isEmpty()))
1022    {
1023      return null;
1024    }
1025
1026    final StringBuilder buffer = new StringBuilder();
1027    final Iterator<String> iterator = l.iterator();
1028    while (iterator.hasNext())
1029    {
1030      buffer.append(iterator.next());
1031      if (iterator.hasNext())
1032      {
1033        buffer.append(EOL);
1034      }
1035    }
1036
1037    return buffer.toString();
1038  }
1039
1040
1041
1042  /**
1043   * {@inheritDoc}
1044   */
1045  @Override()
1046  public LinkedHashMap<String[],String> getExampleUsages()
1047  {
1048    final LinkedHashMap<String[],String> examples =
1049         new LinkedHashMap<>(StaticUtils.computeMapCapacity(2));
1050
1051    String[] args =
1052    {
1053      "--hostname", "server.example.com",
1054      "--port", "389",
1055      "--ldifFile", "data.ldif",
1056      "--rejectFile", "rejects.ldif",
1057      "--numThreads", "4"
1058    };
1059    String description =
1060         "Validate the contents of the 'data.ldif' file using the schema " +
1061         "defined in the specified directory server using four concurrent " +
1062         "threads.  All types of validation will be performed, and " +
1063         "information about any errors will be written to the 'rejects.ldif' " +
1064         "file.";
1065    examples.put(args, description);
1066
1067
1068    args = new String[]
1069    {
1070      "--schemaDirectory", "/ds/config/schema",
1071      "--ldifFile", "data.ldif",
1072      "--rejectFile", "rejects.ldif",
1073      "--ignoreStructuralObjectClasses",
1074      "--ignoreAttributeSyntax"
1075    };
1076    description =
1077         "Validate the contents of the 'data.ldif' file using the schema " +
1078         "defined in LDIF files contained in the /ds/config/schema directory " +
1079         "using a single thread.  Any errors resulting from entries that do " +
1080         "not have exactly one structural object class or from values which " +
1081         "violate the syntax for their associated attribute types will be " +
1082         "ignored.  Information about any other failures will be written to " +
1083         "the 'rejects.ldif' file.";
1084    examples.put(args, description);
1085
1086    return examples;
1087  }
1088
1089
1090
1091  /**
1092   * @return EntryValidator
1093   *
1094   * Returns the EntryValidator
1095   */
1096  public EntryValidator getEntryValidator()
1097  {
1098    return entryValidator;
1099  }
1100}