001/*
002 * Copyright 2013-2019 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright (C) 2013-2019 Ping Identity Corporation
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021package com.unboundid.ldap.sdk.examples;
022
023
024
025import java.io.OutputStream;
026import java.util.Collections;
027import java.util.LinkedHashMap;
028import java.util.List;
029import java.util.Map;
030import java.util.TreeMap;
031import java.util.concurrent.atomic.AtomicLong;
032
033import com.unboundid.asn1.ASN1OctetString;
034import com.unboundid.ldap.sdk.Attribute;
035import com.unboundid.ldap.sdk.DN;
036import com.unboundid.ldap.sdk.Filter;
037import com.unboundid.ldap.sdk.LDAPConnectionOptions;
038import com.unboundid.ldap.sdk.LDAPConnectionPool;
039import com.unboundid.ldap.sdk.LDAPException;
040import com.unboundid.ldap.sdk.LDAPSearchException;
041import com.unboundid.ldap.sdk.ResultCode;
042import com.unboundid.ldap.sdk.SearchRequest;
043import com.unboundid.ldap.sdk.SearchResult;
044import com.unboundid.ldap.sdk.SearchResultEntry;
045import com.unboundid.ldap.sdk.SearchResultReference;
046import com.unboundid.ldap.sdk.SearchResultListener;
047import com.unboundid.ldap.sdk.SearchScope;
048import com.unboundid.ldap.sdk.Version;
049import com.unboundid.ldap.sdk.controls.SimplePagedResultsControl;
050import com.unboundid.util.Debug;
051import com.unboundid.util.LDAPCommandLineTool;
052import com.unboundid.util.StaticUtils;
053import com.unboundid.util.ThreadSafety;
054import com.unboundid.util.ThreadSafetyLevel;
055import com.unboundid.util.args.ArgumentException;
056import com.unboundid.util.args.ArgumentParser;
057import com.unboundid.util.args.DNArgument;
058import com.unboundid.util.args.IntegerArgument;
059import com.unboundid.util.args.StringArgument;
060
061
062
063/**
064 * This class provides a tool that may be used to identify references to entries
065 * that do not exist.  This tool can be useful for verifying existing data in
066 * directory servers that provide support for referential integrity.
067 * <BR><BR>
068 * All of the necessary information is provided using command line arguments.
069 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
070 * class, as well as the following additional arguments:
071 * <UL>
072 *   <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use
073 *       for the searches.  At least one base DN must be provided.</LI>
074 *   <LI>"-A {attribute}" or "--attribute {attribute}" -- specifies an attribute
075 *       that is expected to contain references to other entries.  This
076 *       attribute should be indexed for equality searches, and its values
077 *       should be DNs.  At least one attribute must be provided.</LI>
078 *   <LI>"-z {size}" or "--simplePageSize {size}" -- indicates that the search
079 *       to find entries with references to other entries should use the simple
080 *       paged results control to iterate across entries in fixed-size pages
081 *       rather than trying to use a single search to identify all entries that
082 *       reference other entries.</LI>
083 * </UL>
084 */
085@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
086public final class IdentifyReferencesToMissingEntries
087       extends LDAPCommandLineTool
088       implements SearchResultListener
089{
090  /**
091   * The serial version UID for this serializable class.
092   */
093  private static final long serialVersionUID = 1981894839719501258L;
094
095
096
097  // The number of entries examined so far.
098  private final AtomicLong entriesExamined;
099
100  // The argument used to specify the base DNs to use for searches.
101  private DNArgument baseDNArgument;
102
103  // The argument used to specify the search page size.
104  private IntegerArgument pageSizeArgument;
105
106  // The connection to use for retrieving referenced entries.
107  private LDAPConnectionPool getReferencedEntriesPool;
108
109  // A map with counts of missing references by attribute type.
110  private final Map<String,AtomicLong> missingReferenceCounts;
111
112  // The names of the attributes for which to find missing references.
113  private String[] attributes;
114
115  // The argument used to specify the attributes for which to find missing
116  // references.
117  private StringArgument attributeArgument;
118
119
120
121  /**
122   * Parse the provided command line arguments and perform the appropriate
123   * processing.
124   *
125   * @param  args  The command line arguments provided to this program.
126   */
127  public static void main(final String... args)
128  {
129    final ResultCode resultCode = main(args, System.out, System.err);
130    if (resultCode != ResultCode.SUCCESS)
131    {
132      System.exit(resultCode.intValue());
133    }
134  }
135
136
137
138  /**
139   * Parse the provided command line arguments and perform the appropriate
140   * processing.
141   *
142   * @param  args       The command line arguments provided to this program.
143   * @param  outStream  The output stream to which standard out should be
144   *                    written.  It may be {@code null} if output should be
145   *                    suppressed.
146   * @param  errStream  The output stream to which standard error should be
147   *                    written.  It may be {@code null} if error messages
148   *                    should be suppressed.
149   *
150   * @return A result code indicating whether the processing was successful.
151   */
152  public static ResultCode main(final String[] args,
153                                final OutputStream outStream,
154                                final OutputStream errStream)
155  {
156    final IdentifyReferencesToMissingEntries tool =
157         new IdentifyReferencesToMissingEntries(outStream, errStream);
158    return tool.runTool(args);
159  }
160
161
162
163  /**
164   * Creates a new instance of this tool.
165   *
166   * @param  outStream  The output stream to which standard out should be
167   *                    written.  It may be {@code null} if output should be
168   *                    suppressed.
169   * @param  errStream  The output stream to which standard error should be
170   *                    written.  It may be {@code null} if error messages
171   *                    should be suppressed.
172   */
173  public IdentifyReferencesToMissingEntries(final OutputStream outStream,
174                                            final OutputStream errStream)
175  {
176    super(outStream, errStream);
177
178    baseDNArgument = null;
179    pageSizeArgument = null;
180    attributeArgument = null;
181    getReferencedEntriesPool = null;
182
183    entriesExamined = new AtomicLong(0L);
184    missingReferenceCounts = new TreeMap<>();
185  }
186
187
188
189  /**
190   * Retrieves the name of this tool.  It should be the name of the command used
191   * to invoke this tool.
192   *
193   * @return  The name for this tool.
194   */
195  @Override()
196  public String getToolName()
197  {
198    return "identify-references-to-missing-entries";
199  }
200
201
202
203  /**
204   * Retrieves a human-readable description for this tool.
205   *
206   * @return  A human-readable description for this tool.
207   */
208  @Override()
209  public String getToolDescription()
210  {
211    return "This tool may be used to identify entries containing one or more " +
212         "attributes which reference entries that do not exist.  This may " +
213         "require the ability to perform unindexed searches and/or the " +
214         "ability to use the simple paged results control.";
215  }
216
217
218
219  /**
220   * Retrieves a version string for this tool, if available.
221   *
222   * @return  A version string for this tool, or {@code null} if none is
223   *          available.
224   */
225  @Override()
226  public String getToolVersion()
227  {
228    return Version.NUMERIC_VERSION_STRING;
229  }
230
231
232
233  /**
234   * Indicates whether this tool should provide support for an interactive mode,
235   * in which the tool offers a mode in which the arguments can be provided in
236   * a text-driven menu rather than requiring them to be given on the command
237   * line.  If interactive mode is supported, it may be invoked using the
238   * "--interactive" argument.  Alternately, if interactive mode is supported
239   * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
240   * interactive mode may be invoked by simply launching the tool without any
241   * arguments.
242   *
243   * @return  {@code true} if this tool supports interactive mode, or
244   *          {@code false} if not.
245   */
246  @Override()
247  public boolean supportsInteractiveMode()
248  {
249    return true;
250  }
251
252
253
254  /**
255   * Indicates whether this tool defaults to launching in interactive mode if
256   * the tool is invoked without any command-line arguments.  This will only be
257   * used if {@link #supportsInteractiveMode()} returns {@code true}.
258   *
259   * @return  {@code true} if this tool defaults to using interactive mode if
260   *          launched without any command-line arguments, or {@code false} if
261   *          not.
262   */
263  @Override()
264  public boolean defaultsToInteractiveMode()
265  {
266    return true;
267  }
268
269
270
271  /**
272   * Indicates whether this tool should provide arguments for redirecting output
273   * to a file.  If this method returns {@code true}, then the tool will offer
274   * an "--outputFile" argument that will specify the path to a file to which
275   * all standard output and standard error content will be written, and it will
276   * also offer a "--teeToStandardOut" argument that can only be used if the
277   * "--outputFile" argument is present and will cause all output to be written
278   * to both the specified output file and to standard output.
279   *
280   * @return  {@code true} if this tool should provide arguments for redirecting
281   *          output to a file, or {@code false} if not.
282   */
283  @Override()
284  protected boolean supportsOutputFile()
285  {
286    return true;
287  }
288
289
290
291  /**
292   * Indicates whether this tool should default to interactively prompting for
293   * the bind password if a password is required but no argument was provided
294   * to indicate how to get the password.
295   *
296   * @return  {@code true} if this tool should default to interactively
297   *          prompting for the bind password, or {@code false} if not.
298   */
299  @Override()
300  protected boolean defaultToPromptForBindPassword()
301  {
302    return true;
303  }
304
305
306
307  /**
308   * Indicates whether this tool supports the use of a properties file for
309   * specifying default values for arguments that aren't specified on the
310   * command line.
311   *
312   * @return  {@code true} if this tool supports the use of a properties file
313   *          for specifying default values for arguments that aren't specified
314   *          on the command line, or {@code false} if not.
315   */
316  @Override()
317  public boolean supportsPropertiesFile()
318  {
319    return true;
320  }
321
322
323
324  /**
325   * Indicates whether the LDAP-specific arguments should include alternate
326   * versions of all long identifiers that consist of multiple words so that
327   * they are available in both camelCase and dash-separated versions.
328   *
329   * @return  {@code true} if this tool should provide multiple versions of
330   *          long identifiers for LDAP-specific arguments, or {@code false} if
331   *          not.
332   */
333  @Override()
334  protected boolean includeAlternateLongIdentifiers()
335  {
336    return true;
337  }
338
339
340
341  /**
342   * Indicates whether this tool should provide a command-line argument that
343   * allows for low-level SSL debugging.  If this returns {@code true}, then an
344   * "--enableSSLDebugging}" argument will be added that sets the
345   * "javax.net.debug" system property to "all" before attempting any
346   * communication.
347   *
348   * @return  {@code true} if this tool should offer an "--enableSSLDebugging"
349   *          argument, or {@code false} if not.
350   */
351  @Override()
352  protected boolean supportsSSLDebugging()
353  {
354    return true;
355  }
356
357
358
359  /**
360   * Adds the arguments needed by this command-line tool to the provided
361   * argument parser which are not related to connecting or authenticating to
362   * the directory server.
363   *
364   * @param  parser  The argument parser to which the arguments should be added.
365   *
366   * @throws  ArgumentException  If a problem occurs while adding the arguments.
367   */
368  @Override()
369  public void addNonLDAPArguments(final ArgumentParser parser)
370         throws ArgumentException
371  {
372    String description = "The search base DN(s) to use to find entries with " +
373         "references to other entries.  At least one base DN must be " +
374         "specified.";
375    baseDNArgument = new DNArgument('b', "baseDN", true, 0, "{dn}",
376         description);
377    baseDNArgument.addLongIdentifier("base-dn", true);
378    parser.addArgument(baseDNArgument);
379
380    description = "The attribute(s) for which to find missing references.  " +
381         "At least one attribute must be specified, and each attribute " +
382         "must be indexed for equality searches and have values which are DNs.";
383    attributeArgument = new StringArgument('A', "attribute", true, 0, "{attr}",
384         description);
385    parser.addArgument(attributeArgument);
386
387    description = "The maximum number of entries to retrieve at a time when " +
388         "attempting to find entries with references to other entries.  This " +
389         "requires that the authenticated user have permission to use the " +
390         "simple paged results control, but it can avoid problems with the " +
391         "server sending entries too quickly for the client to handle.  By " +
392         "default, the simple paged results control will not be used.";
393    pageSizeArgument =
394         new IntegerArgument('z', "simplePageSize", false, 1, "{num}",
395              description, 1, Integer.MAX_VALUE);
396    pageSizeArgument.addLongIdentifier("simple-page-size", true);
397    parser.addArgument(pageSizeArgument);
398  }
399
400
401
402  /**
403   * Retrieves the connection options that should be used for connections that
404   * are created with this command line tool.  Subclasses may override this
405   * method to use a custom set of connection options.
406   *
407   * @return  The connection options that should be used for connections that
408   *          are created with this command line tool.
409   */
410  @Override()
411  public LDAPConnectionOptions getConnectionOptions()
412  {
413    final LDAPConnectionOptions options = new LDAPConnectionOptions();
414
415    options.setUseSynchronousMode(true);
416    options.setResponseTimeoutMillis(0L);
417
418    return options;
419  }
420
421
422
423  /**
424   * Performs the core set of processing for this tool.
425   *
426   * @return  A result code that indicates whether the processing completed
427   *          successfully.
428   */
429  @Override()
430  public ResultCode doToolProcessing()
431  {
432    // Establish a connection to the target directory server to use for
433    // finding references to entries.
434    final LDAPConnectionPool findReferencesPool;
435    try
436    {
437      findReferencesPool = getConnectionPool(1, 1);
438      findReferencesPool.setRetryFailedOperationsDueToInvalidConnections(true);
439    }
440    catch (final LDAPException le)
441    {
442      Debug.debugException(le);
443      err("Unable to establish a connection to the directory server:  ",
444           StaticUtils.getExceptionMessage(le));
445      return le.getResultCode();
446    }
447
448    try
449    {
450      // Establish a second connection to use for retrieving referenced entries.
451      try
452      {
453        getReferencedEntriesPool = getConnectionPool(1,1);
454        getReferencedEntriesPool.
455             setRetryFailedOperationsDueToInvalidConnections(true);
456      }
457      catch (final LDAPException le)
458      {
459        Debug.debugException(le);
460        err("Unable to establish a connection to the directory server:  ",
461             StaticUtils.getExceptionMessage(le));
462        return le.getResultCode();
463      }
464
465
466      // Get the set of attributes for which to find missing references.
467      final List<String> attrList = attributeArgument.getValues();
468      attributes = new String[attrList.size()];
469      attrList.toArray(attributes);
470
471
472      // Construct a search filter that will be used to find all entries with
473      // references to other entries.
474      final Filter filter;
475      if (attributes.length == 1)
476      {
477        filter = Filter.createPresenceFilter(attributes[0]);
478        missingReferenceCounts.put(attributes[0], new AtomicLong(0L));
479      }
480      else
481      {
482        final Filter[] orComps = new Filter[attributes.length];
483        for (int i=0; i < attributes.length; i++)
484        {
485          orComps[i] = Filter.createPresenceFilter(attributes[i]);
486          missingReferenceCounts.put(attributes[i], new AtomicLong(0L));
487        }
488        filter = Filter.createORFilter(orComps);
489      }
490
491
492      // Iterate across all of the search base DNs and perform searches to find
493      // missing references.
494      for (final DN baseDN : baseDNArgument.getValues())
495      {
496        ASN1OctetString cookie = null;
497        do
498        {
499          final SearchRequest searchRequest = new SearchRequest(this,
500               baseDN.toString(), SearchScope.SUB, filter, attributes);
501          if (pageSizeArgument.isPresent())
502          {
503            searchRequest.addControl(new SimplePagedResultsControl(
504                 pageSizeArgument.getValue(), cookie, false));
505          }
506
507          SearchResult searchResult;
508          try
509          {
510            searchResult = findReferencesPool.search(searchRequest);
511          }
512          catch (final LDAPSearchException lse)
513          {
514            Debug.debugException(lse);
515            try
516            {
517              searchResult = findReferencesPool.search(searchRequest);
518            }
519            catch (final LDAPSearchException lse2)
520            {
521              Debug.debugException(lse2);
522              searchResult = lse2.getSearchResult();
523            }
524          }
525
526          if (searchResult.getResultCode() != ResultCode.SUCCESS)
527          {
528            err("An error occurred while attempting to search for missing " +
529                 "references to entries below " + baseDN + ":  " +
530                 searchResult.getDiagnosticMessage());
531            return searchResult.getResultCode();
532          }
533
534          final SimplePagedResultsControl pagedResultsResponse;
535          try
536          {
537            pagedResultsResponse = SimplePagedResultsControl.get(searchResult);
538          }
539          catch (final LDAPException le)
540          {
541            Debug.debugException(le);
542            err("An error occurred while attempting to decode a simple " +
543                 "paged results response control in the response to a " +
544                 "search for entries below " + baseDN + ":  " +
545                 StaticUtils.getExceptionMessage(le));
546            return le.getResultCode();
547          }
548
549          if (pagedResultsResponse != null)
550          {
551            if (pagedResultsResponse.moreResultsToReturn())
552            {
553              cookie = pagedResultsResponse.getCookie();
554            }
555            else
556            {
557              cookie = null;
558            }
559          }
560        }
561        while (cookie != null);
562      }
563
564
565      // See if there were any missing references found.
566      boolean missingReferenceFound = false;
567      for (final Map.Entry<String,AtomicLong> e :
568           missingReferenceCounts.entrySet())
569      {
570        final long numMissing = e.getValue().get();
571        if (numMissing > 0L)
572        {
573          if (! missingReferenceFound)
574          {
575            err();
576            missingReferenceFound = true;
577          }
578
579          err("Found " + numMissing + ' ' + e.getKey() +
580               " references to entries that do not exist.");
581        }
582      }
583
584      if (missingReferenceFound)
585      {
586        return ResultCode.CONSTRAINT_VIOLATION;
587      }
588      else
589      {
590        out("No references were found to entries that do not exist.");
591        return ResultCode.SUCCESS;
592      }
593    }
594    finally
595    {
596      findReferencesPool.close();
597
598      if (getReferencedEntriesPool != null)
599      {
600        getReferencedEntriesPool.close();
601      }
602    }
603  }
604
605
606
607  /**
608   * Retrieves a map that correlates the number of missing references found by
609   * attribute type.
610   *
611   * @return  A map that correlates the number of missing references found by
612   *          attribute type.
613   */
614  public Map<String,AtomicLong> getMissingReferenceCounts()
615  {
616    return Collections.unmodifiableMap(missingReferenceCounts);
617  }
618
619
620
621  /**
622   * Retrieves a set of information that may be used to generate example usage
623   * information.  Each element in the returned map should consist of a map
624   * between an example set of arguments and a string that describes the
625   * behavior of the tool when invoked with that set of arguments.
626   *
627   * @return  A set of information that may be used to generate example usage
628   *          information.  It may be {@code null} or empty if no example usage
629   *          information is available.
630   */
631  @Override()
632  public LinkedHashMap<String[],String> getExampleUsages()
633  {
634    final LinkedHashMap<String[],String> exampleMap =
635         new LinkedHashMap<>(StaticUtils.computeMapCapacity(1));
636
637    final String[] args =
638    {
639      "--hostname", "server.example.com",
640      "--port", "389",
641      "--bindDN", "uid=john.doe,ou=People,dc=example,dc=com",
642      "--bindPassword", "password",
643      "--baseDN", "dc=example,dc=com",
644      "--attribute", "member",
645      "--attribute", "uniqueMember",
646      "--simplePageSize", "100"
647    };
648    exampleMap.put(args,
649         "Identify all entries below dc=example,dc=com in which either the " +
650              "member or uniqueMember attribute references an entry that " +
651              "does not exist.");
652
653    return exampleMap;
654  }
655
656
657
658  /**
659   * Indicates that the provided search result entry has been returned by the
660   * server and may be processed by this search result listener.
661   *
662   * @param  searchEntry  The search result entry that has been returned by the
663   *                      server.
664   */
665  @Override()
666  public void searchEntryReturned(final SearchResultEntry searchEntry)
667  {
668    try
669    {
670      // Find attributes which references to entries that do not exist.
671      for (final String attr : attributes)
672      {
673        final List<Attribute> attrList =
674             searchEntry.getAttributesWithOptions(attr, null);
675        for (final Attribute a : attrList)
676        {
677          for (final String value : a.getValues())
678          {
679            try
680            {
681              final SearchResultEntry e =
682                   getReferencedEntriesPool.getEntry(value, "1.1");
683              if (e == null)
684              {
685                err("Entry '", searchEntry.getDN(), "' includes attribute ",
686                     a.getName(), " that references entry '", value,
687                     "' which does not exist.");
688                missingReferenceCounts.get(attr).incrementAndGet();
689              }
690            }
691            catch (final LDAPException le)
692            {
693              Debug.debugException(le);
694              err("An error occurred while attempting to determine whether " +
695                   "entry '" + value + "' referenced in attribute " +
696                   a.getName() + " of entry '" + searchEntry.getDN() +
697                   "' exists:  " + StaticUtils.getExceptionMessage(le));
698              missingReferenceCounts.get(attr).incrementAndGet();
699            }
700          }
701        }
702      }
703    }
704    finally
705    {
706      final long count = entriesExamined.incrementAndGet();
707      if ((count % 1000L) == 0L)
708      {
709        out(count, " entries examined");
710      }
711    }
712  }
713
714
715
716  /**
717   * Indicates that the provided search result reference has been returned by
718   * the server and may be processed by this search result listener.
719   *
720   * @param  searchReference  The search result reference that has been returned
721   *                          by the server.
722   */
723  @Override()
724  public void searchReferenceReturned(
725                   final SearchResultReference searchReference)
726  {
727    // No implementation is required.  This tool will not follow referrals.
728  }
729}